github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-23 22:36:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add server and worker resource request and limits (#833)

Browse Source 
Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
This commit is contained in:
Hussein Galal
2026-04-30 16:25:06 +03:00
committed by GitHub
parent f9f98e109a
commit 06ba7da0ca
10 changed files with 339 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
charts/k3k/templates/crds/k3k.io_clusters.yaml
+118
View File
@@ -638,6 +638,65 @@ spec:
x-kubernetes-int-or-string: true
description: ServerLimit specifies resource limits for server nodes.
type: object
serverResources:
description: ServerResources specifies resources limits and requests for server nodes.
properties:
claims:
description: |-
Claims lists the names of resources, defined in spec.resourceClaims,
that are used by this container.
This is an alpha field and requires enabling the
DynamicResourceAllocation feature gate.
This field is immutable. It can only be set for containers.
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: |-
Name must match the name of one entry in pod.spec.resourceClaims of
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Limits describes the maximum amount of compute resources allowed.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Requests describes the minimum amount of compute resources required.
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
otherwise to an implementation-defined value. Requests cannot exceed Limits.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
type: object
servers:
default: 1
description: |-
@@ -807,6 +866,65 @@ spec:
x-kubernetes-int-or-string: true
description: WorkerLimit specifies resource limits for agent nodes.
type: object
workerResources:
description: WorkerResources specifies resources limits and requests for worker nodes.
properties:
claims:
description: |-
Claims lists the names of resources, defined in spec.resourceClaims,
that are used by this container.
This is an alpha field and requires enabling the
DynamicResourceAllocation feature gate.
This field is immutable. It can only be set for containers.
items:
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
properties:
name:
description: |-
Name must match the name of one entry in pod.spec.resourceClaims of
the Pod where this field is used. It makes that resource available
inside a container.
type: string
request:
description: |-
Request is the name chosen for a request in the referenced claim.
If empty, everything from the claim is made available, otherwise
only the result of this request.
type: string
required:
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
limits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Limits describes the maximum amount of compute resources allowed.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
requests:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
Requests describes the minimum amount of compute resources required.
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
otherwise to an implementation-defined value. Requests cannot exceed Limits.
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
type: object
type: object
type: object
status:
default: {}
docs/crds/crds.adoc
+2
View File
@@ -201,6 +201,8 @@ Example: ["--node-name=my-agent-node"] + | |
| *`addons`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-addon[$$Addon$$] array__ | Addons specifies secrets containing raw YAML to deploy on cluster startup. + | |
| *`serverLimit`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core[$$ResourceList$$]__ | ServerLimit specifies resource limits for server nodes. + | |
| *`workerLimit`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core[$$ResourceList$$]__ | WorkerLimit specifies resource limits for agent nodes. + | |
| *`serverResources`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core[$$ResourceRequirements$$]__ | ServerResources specifies resources limits and requests for server nodes. + | |
| *`workerResources`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core[$$ResourceRequirements$$]__ | WorkerResources specifies resources limits and requests for worker nodes. + | |
| *`mirrorHostNodes`* __boolean__ | MirrorHostNodes controls whether node objects from the host cluster +
are mirrored into the virtual cluster. + | |
| *`customCAs`* __xref:{anchor_prefix}-github-com-rancher-k3k-pkg-apis-k3k-io-v1beta1-customcas[$$CustomCAs$$]__ | CustomCAs specifies the cert/key pairs for custom CA certificates. + | |
docs/crds/crds.md
+2
View File
@@ -150,6 +150,8 @@ _Appears in:_
| `addons` _[Addon](#addon) array_ | Addons specifies secrets containing raw YAML to deploy on cluster startup. | | |
| `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit specifies resource limits for server nodes. | | |
| `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit specifies resource limits for agent nodes. | | |
| `serverResources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core)_ | ServerResources specifies resources limits and requests for server nodes. | | |
| `workerResources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcerequirements-v1-core)_ | WorkerResources specifies resources limits and requests for worker nodes. | | |
| `mirrorHostNodes` _boolean_ | MirrorHostNodes controls whether node objects from the host cluster<br />are mirrored into the virtual cluster. | | |
| `customCAs` _[CustomCAs](#customcas)_ | CustomCAs specifies the cert/key pairs for custom CA certificates. | | |
| `sync` _[SyncConfig](#syncconfig)_ | Sync specifies the resources types that will be synced from virtual cluster to host cluster. | \{ \} | |
pkg/apis/k3k.io/v1beta1/types.go
+10
View File
@@ -169,6 +169,16 @@ type ClusterSpec struct {
// +optional
WorkerLimit corev1.ResourceList `json:"workerLimit,omitempty"`
// ServerResources specifies resources limits and requests for server nodes.
//
// +optional
ServerResources *corev1.ResourceRequirements `json:"serverResources,omitempty"`
// WorkerResources specifies resources limits and requests for worker nodes.
//
// +optional
WorkerResources *corev1.ResourceRequirements `json:"workerResources,omitempty"`
// MirrorHostNodes controls whether node objects from the host cluster
// are mirrored into the virtual cluster.
//
pkg/apis/k3k.io/v1beta1/zz_generated.deepcopy.go
+10
View File
@@ -190,6 +190,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
(*out)[key] = val.DeepCopy()
}
}
if in.ServerResources != nil {
in, out := &in.ServerResources, &out.ServerResources
*out = new(v1.ResourceRequirements)
(*in).DeepCopyInto(*out)
}
if in.WorkerResources != nil {
in, out := &in.WorkerResources, &out.WorkerResources
*out = new(v1.ResourceRequirements)
(*in).DeepCopyInto(*out)
}
if in.CustomCAs != nil {
in, out := &in.CustomCAs, &out.CustomCAs
*out = new(CustomCAs)
pkg/controller/cluster/agent/shared.go
+7
View File
@@ -271,6 +271,13 @@ func (s *SharedAgent) podSpec() v1.PodSpec {
}
}
// specifying WorkerResources will take precedence over WorkerLimits
if s.cluster.Spec.WorkerResources != nil {
// removing container previous limit
podSpec.Containers[0].Resources = v1.ResourceRequirements{}
podSpec.Resources = s.cluster.Spec.WorkerResources
}
return podSpec
}
pkg/controller/cluster/agent/shared_test.go
+89
View File
@@ -302,6 +302,95 @@ func Test_sharedAgentPodSpec(t *testing.T) {
},
}
return spec
},
},
{
name: "worker resources sets pods resource limits/requests",
sharedAgent: SharedAgent{
Config: &Config{
cluster: &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "sc-workerResources",
Namespace: "shared-test",
},
Spec: v1beta1.ClusterSpec{
WorkerResources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
},
},
image: "rancher/k3k-kubelet:latest",
kubeletPort: 10250,
},
expectedPodSpec: func(sa SharedAgent) corev1.PodSpec {
spec := baseSharedAgentPodSpec(sa)
spec.Resources = &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
}
return spec
},
},
{
name: "worker resources takes precedence over WorkerLimit",
sharedAgent: SharedAgent{
Config: &Config{
cluster: &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "sc-workerResources",
Namespace: "shared-test",
},
Spec: v1beta1.ClusterSpec{
WorkerLimit: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
WorkerResources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
},
},
image: "rancher/k3k-kubelet:latest",
kubeletPort: 10250,
},
expectedPodSpec: func(sa SharedAgent) corev1.PodSpec {
spec := baseSharedAgentPodSpec(sa)
spec.Resources = &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
}
spec.Containers[0].Resources = corev1.ResourceRequirements{}
return spec
},
},
pkg/controller/cluster/agent/virtual.go
+7
View File
@@ -262,6 +262,13 @@ func (v *VirtualAgent) podSpec(image, name string) v1.PodSpec {
}
}
// specifying WorkerResources will take precedence over WorkerLimits
if v.cluster.Spec.WorkerResources != nil {
// removing container previous limit
podSpec.Containers[0].Resources = v1.ResourceRequirements{}
podSpec.Resources = v.cluster.Spec.WorkerResources
}
for _, imagePullSecret := range v.imagePullSecrets {
podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, v1.LocalObjectReference{Name: imagePullSecret})
}
pkg/controller/cluster/agent/virtual_test.go
+87
View File
@@ -349,6 +349,93 @@ func Test_virtualAgentPodSpec(t *testing.T) {
},
}
return spec
},
},
{
name: "worker resources sets pods resource limits/requests",
virtualAgent: VirtualAgent{
Config: &Config{
cluster: &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "sc-workerResources",
Namespace: "shared-test",
},
Spec: v1beta1.ClusterSpec{
WorkerResources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
},
},
Image: "rancher/k3k:latest",
},
expectedPodSpec: func(va VirtualAgent) corev1.PodSpec {
spec := baseVirtualAgentPodSpec(va)
spec.Resources = &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
}
return spec
},
},
{
name: "worker resources takes precedence over WorkerLimit",
virtualAgent: VirtualAgent{
Config: &Config{
cluster: &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "sc-workerResources",
Namespace: "shared-test",
},
Spec: v1beta1.ClusterSpec{
WorkerLimit: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
WorkerResources: &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
},
},
Image: "rancher/k3k:latest",
},
expectedPodSpec: func(va VirtualAgent) corev1.PodSpec {
spec := baseVirtualAgentPodSpec(va)
spec.Resources = &corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
}
spec.Containers[0].Resources = corev1.ResourceRequirements{}
return spec
},
},
pkg/controller/cluster/server/server.go
+7
View File
@@ -242,6 +242,13 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string)
}
}
// specifying ServerResources will take precedence over ServerLimits
if s.cluster.Spec.ServerResources != nil {
// removing container previous limit
podSpec.Containers[0].Resources = v1.ResourceRequirements{}
podSpec.Resources = s.cluster.Spec.ServerResources
}
podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, s.cluster.Spec.ServerEnvs...)
// add image pull secrets