github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-23 22:36:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial support for ResourceQuotas in clustersets (#308)

Browse Source 
* Add ResourceQuota to clusterset

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* wsl fixes

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Generate docs

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix tests

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Add a defualt limitRange for ClusterSets

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix linting

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix docs

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Add test for clusterset limitRange

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Add server and worker limits

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* make charts

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* add default limits and fixes to resourcesquota

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* wsl lint

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* wsl lint

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* make docs

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* make build-crds

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* make build-crds

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* make spec as pointer

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix tests

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix tests

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* delete default limit

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* wsl lint

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* wsl

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* wsl

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Update tests

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* Update tests

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* return on delete in limitrange

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* wsl

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

---------

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
This commit is contained in:
Hussein Galal
2025-04-03 12:30:48 +02:00
committed by GitHub
parent b58578788c
commit 0843a9e313
9 changed files with 478 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
charts/k3k/crds/k3k.io_clusters.yaml
+18 -23
View File
@@ -94,29 +94,6 @@ spec:
x-kubernetes-validations:
- message: clusterDNS is immutable
rule: self == oldSelf
clusterLimit:
description: Limit defines resource limits for server/agent nodes.
properties:
serverLimit:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: ServerLimit specifies resource limits for server
nodes.
type: object
workerLimit:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: WorkerLimit specifies resource limits for agent nodes.
type: object
type: object
expose:
description: |-
Expose specifies options for exposing the API server.
@@ -225,6 +202,15 @@ spec:
items:
type: string
type: array
serverLimit:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: ServerLimit specifies resource limits for server nodes.
type: object
servers:
default: 1
description: |-
@@ -271,6 +257,15 @@ spec:
It should follow the K3s versioning convention (e.g., v1.28.2-k3s1).
If not specified, the Kubernetes version of the host node will be used.
type: string
workerLimit:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: WorkerLimit specifies resource limits for agent nodes.
type: object
type: object
status:
description: Status reflects the observed state of the Cluster.
charts/k3k/crds/k3k.io_clustersets.yaml
+139 -33
View File
@@ -59,30 +59,6 @@ spec:
x-kubernetes-validations:
- message: mode is immutable
rule: self == oldSelf
defaultLimits:
description: DefaultLimits specifies the default resource limits for
servers/agents when a cluster in the set doesn't provide any.
properties:
serverLimit:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: ServerLimit specifies resource limits for server
nodes.
type: object
workerLimit:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: WorkerLimit specifies resource limits for agent nodes.
type: object
type: object
defaultNodeSelector:
additionalProperties:
type: string
@@ -97,15 +73,81 @@ spec:
description: DisableNetworkPolicy indicates whether to disable the
creation of a default network policy for cluster isolation.
type: boolean
maxLimits:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: MaxLimits specifies the maximum resource limits that
apply to all clusters (server + agent) in the set.
limit:
description: |-
Limit specifies the LimitRange that will be applied to all pods within the ClusterSet
to set defaults and constraints (min/max)
properties:
limits:
description: Limits is the list of LimitRangeItem objects that
are enforced.
items:
description: LimitRangeItem defines a min/max usage limit for
any resource that matches on kind.
properties:
default:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: Default resource requirement limit value by
resource name if resource limit is omitted.
type: object
defaultRequest:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: DefaultRequest is the default resource requirement
request value by resource name if resource request is
omitted.
type: object
max:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: Max usage constraints on this kind by resource
name.
type: object
maxLimitRequestRatio:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: MaxLimitRequestRatio if specified, the named
resource must have a request and limit that are both non-zero
where limit divided by request is less than or equal to
the enumerated value; this represents the max burst for
the named resource.
type: object
min:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: Min usage constraints on this kind by resource
name.
type: object
type:
description: Type of resource that this limit applies to.
type: string
required:
- type
type: object
type: array
required:
- limits
type: object
podSecurityAdmissionLevel:
description: PodSecurityAdmissionLevel specifies the pod security
@@ -115,6 +157,70 @@ spec:
- baseline
- restricted
type: string
quota:
description: Quota specifies the resource limits for clusters within
a clusterset.
properties:
hard:
additionalProperties:
anyOf:
- type: integer
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: |-
hard is the set of desired hard limits for each named resource.
More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/
type: object
scopeSelector:
description: |-
scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota
but expressed using ScopeSelectorOperator in combination with possible values.
For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched.
properties:
matchExpressions:
description: A list of scope selector requirements by scope
of the resources.
items:
description: |-
A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator
that relates the scope name and values.
properties:
operator:
description: |-
Represents a scope's relationship to a set of values.
Valid operators are In, NotIn, Exists, DoesNotExist.
type: string
scopeName:
description: The name of the scope that the selector
applies to.
type: string
values:
description: |-
An array of string values. If the operator is In or NotIn,
the values array must be non-empty. If the operator is Exists or DoesNotExist,
the values array must be empty.
This array is replaced during a strategic merge patch.
items:
type: string
type: array
required:
- operator
- scopeName
type: object
type: array
type: object
x-kubernetes-map-type: atomic
scopes:
description: |-
A collection of filters that must match each object tracked by a quota.
If not specified, the quota matches all objects.
items:
description: A ResourceQuotaScope defines a filter that must
match each object tracked by a quota
type: string
type: array
type: object
type: object
status:
description: Status reflects the observed state of the ClusterSet.
docs/crds/crd-docs.md
+2 -18
View File
@@ -51,23 +51,6 @@ _Appears in:_
| `spec` _[ClusterSpec](#clusterspec)_ | Spec defines the desired state of the Cluster. | \{ \} | |
#### ClusterLimit
ClusterLimit defines resource limits for server and agent nodes.
_Appears in:_
- [ClusterSpec](#clusterspec)
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
| `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit specifies resource limits for server nodes. | | |
| `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit specifies resource limits for agent nodes. | | |
#### ClusterList
@@ -124,12 +107,13 @@ _Appears in:_
| `expose` _[ExposeConfig](#exposeconfig)_ | Expose specifies options for exposing the API server.<br />By default, it's only exposed as a ClusterIP. | | |
| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector specifies node labels to constrain where server/agent pods are scheduled.<br />In "shared" mode, this also applies to workloads. | | |
| `priorityClass` _string_ | PriorityClass specifies the priorityClassName for server/agent pods.<br />In "shared" mode, this also applies to workloads. | | |
| `clusterLimit` _[ClusterLimit](#clusterlimit)_ | Limit defines resource limits for server/agent nodes. | | |
| `tokenSecretRef` _[SecretReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#secretreference-v1-core)_ | TokenSecretRef is a Secret reference containing the token used by worker nodes to join the cluster.<br />The Secret must have a "token" field in its data. | | |
| `tlsSANs` _string array_ | TLSSANs specifies subject alternative names for the K3s server certificate. | | |
| `serverArgs` _string array_ | ServerArgs specifies ordered key-value pairs for K3s server pods.<br />Example: ["--tls-san=example.com"] | | |
| `agentArgs` _string array_ | AgentArgs specifies ordered key-value pairs for K3s agent pods.<br />Example: ["--node-name=my-agent-node"] | | |
| `addons` _[Addon](#addon) array_ | Addons specifies secrets containing raw YAML to deploy on cluster startup. | | |
| `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit specifies resource limits for server nodes. | | |
| `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit specifies resource limits for agent nodes. | | |
pkg/apis/k3k.io/v1alpha1/types.go
+18 -21
View File
@@ -114,11 +114,6 @@ type ClusterSpec struct {
// +optional
PriorityClass string `json:"priorityClass,omitempty"`
// Limit defines resource limits for server/agent nodes.
//
// +optional
Limit *ClusterLimit `json:"clusterLimit,omitempty"`
// TokenSecretRef is a Secret reference containing the token used by worker nodes to join the cluster.
// The Secret must have a "token" field in its data.
//
@@ -146,6 +141,16 @@ type ClusterSpec struct {
//
// +optional
Addons []Addon `json:"addons,omitempty"`
// ServerLimit specifies resource limits for server nodes.
//
// +optional
ServerLimit v1.ResourceList `json:"serverLimit,omitempty"`
// WorkerLimit specifies resource limits for agent nodes.
//
// +optional
WorkerLimit v1.ResourceList `json:"workerLimit,omitempty"`
}
// ClusterMode is the possible provisioning mode of a Cluster.
@@ -175,15 +180,6 @@ const (
DynamicPersistenceMode = PersistenceMode("dynamic")
)
// ClusterLimit defines resource limits for server and agent nodes.
type ClusterLimit struct {
// ServerLimit specifies resource limits for server nodes.
ServerLimit v1.ResourceList `json:"serverLimit,omitempty"`
// WorkerLimit specifies resource limits for agent nodes.
WorkerLimit v1.ResourceList `json:"workerLimit,omitempty"`
}
// Addon specifies a Secret containing YAML to be deployed on cluster startup.
type Addon struct {
// SecretNamespace is the namespace of the Secret.
@@ -338,10 +334,16 @@ type ClusterSet struct {
// ClusterSetSpec defines the desired state of a ClusterSet.
type ClusterSetSpec struct {
// DefaultLimits specifies the default resource limits for servers/agents when a cluster in the set doesn't provide any.
// Quota specifies the resource limits for clusters within a clusterset.
//
// +optional
DefaultLimits *ClusterLimit `json:"defaultLimits,omitempty"`
Quota *v1.ResourceQuotaSpec `json:"quota,omitempty"`
// Limit specifies the LimitRange that will be applied to all pods within the ClusterSet
// to set defaults and constraints (min/max)
//
// +optional
Limit *v1.LimitRangeSpec `json:"limit,omitempty"`
// DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the set.
//
@@ -353,11 +355,6 @@ type ClusterSetSpec struct {
// +optional
DefaultPriorityClass string `json:"defaultPriorityClass,omitempty"`
// MaxLimits specifies the maximum resource limits that apply to all clusters (server + agent) in the set.
//
// +optional
MaxLimits v1.ResourceList `json:"maxLimits,omitempty"`
// AllowedModeTypes specifies the allowed cluster provisioning modes. Defaults to [shared].
//
// +kubebuilder:default={shared}
pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go
+31 -54
View File
@@ -55,36 +55,6 @@ func (in *Cluster) DeepCopyObject() runtime.Object {
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterLimit) DeepCopyInto(out *ClusterLimit) {
*out = *in
if in.ServerLimit != nil {
in, out := &in.ServerLimit, &out.ServerLimit
*out = make(v1.ResourceList, len(*in))
for key, val := range *in {
(*out)[key] = val.DeepCopy()
}
}
if in.WorkerLimit != nil {
in, out := &in.WorkerLimit, &out.WorkerLimit
*out = make(v1.ResourceList, len(*in))
for key, val := range *in {
(*out)[key] = val.DeepCopy()
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLimit.
func (in *ClusterLimit) DeepCopy() *ClusterLimit {
if in == nil {
return nil
}
out := new(ClusterLimit)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterList) DeepCopyInto(out *ClusterList) {
*out = *in
@@ -182,16 +152,14 @@ func (in *ClusterSetList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ClusterSetSpec) DeepCopyInto(out *ClusterSetSpec) {
*out = *in
if in.MaxLimits != nil {
in, out := &in.MaxLimits, &out.MaxLimits
*out = make(v1.ResourceList, len(*in))
for key, val := range *in {
(*out)[key] = val.DeepCopy()
}
if in.Quota != nil {
in, out := &in.Quota, &out.Quota
*out = new(v1.ResourceQuotaSpec)
(*in).DeepCopyInto(*out)
}
if in.DefaultLimits != nil {
in, out := &in.DefaultLimits, &out.DefaultLimits
*out = new(ClusterLimit)
if in.Limit != nil {
in, out := &in.Limit, &out.Limit
*out = new(v1.LimitRangeSpec)
(*in).DeepCopyInto(*out)
}
if in.DefaultNodeSelector != nil {
@@ -260,6 +228,12 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
*out = new(int32)
**out = **in
}
in.Persistence.DeepCopyInto(&out.Persistence)
if in.Expose != nil {
in, out := &in.Expose, &out.Expose
*out = new(ExposeConfig)
(*in).DeepCopyInto(*out)
}
if in.NodeSelector != nil {
in, out := &in.NodeSelector, &out.NodeSelector
*out = make(map[string]string, len(*in))
@@ -267,16 +241,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
(*out)[key] = val
}
}
if in.Limit != nil {
in, out := &in.Limit, &out.Limit
*out = new(ClusterLimit)
(*in).DeepCopyInto(*out)
}
if in.TokenSecretRef != nil {
in, out := &in.TokenSecretRef, &out.TokenSecretRef
*out = new(v1.SecretReference)
**out = **in
}
if in.TLSSANs != nil {
in, out := &in.TLSSANs, &out.TLSSANs
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.ServerArgs != nil {
in, out := &in.ServerArgs, &out.ServerArgs
*out = make([]string, len(*in))
@@ -287,21 +261,24 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.TLSSANs != nil {
in, out := &in.TLSSANs, &out.TLSSANs
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Addons != nil {
in, out := &in.Addons, &out.Addons
*out = make([]Addon, len(*in))
copy(*out, *in)
}
in.Persistence.DeepCopyInto(&out.Persistence)
if in.Expose != nil {
in, out := &in.Expose, &out.Expose
*out = new(ExposeConfig)
(*in).DeepCopyInto(*out)
if in.ServerLimit != nil {
in, out := &in.ServerLimit, &out.ServerLimit
*out = make(v1.ResourceList, len(*in))
for key, val := range *in {
(*out)[key] = val.DeepCopy()
}
}
if in.WorkerLimit != nil {
in, out := &in.WorkerLimit, &out.WorkerLimit
*out = make(v1.ResourceList, len(*in))
for key, val := range *in {
(*out)[key] = val.DeepCopy()
}
}
return
}
pkg/controller/cluster/agent/virtual.go
+7
View File
@@ -228,5 +228,12 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect
},
}
// specify resource limits if specified for the servers.
if v.cluster.Spec.WorkerLimit != nil {
podSpec.Containers[0].Resources = v1.ResourceRequirements{
Limits: v.cluster.Spec.WorkerLimit,
}
}
return podSpec
}
pkg/controller/cluster/server/server.go
+7 -8
View File
@@ -46,11 +46,6 @@ func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string) *S
}
func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) v1.PodSpec {
var limit v1.ResourceList
if s.cluster.Spec.Limit != nil && s.cluster.Spec.Limit.ServerLimit != nil {
limit = s.cluster.Spec.Limit.ServerLimit
}
podSpec := v1.PodSpec{
NodeSelector: s.cluster.Spec.NodeSelector,
PriorityClassName: s.cluster.Spec.PriorityClass,
@@ -118,9 +113,6 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string)
{
Name: name,
Image: image,
Resources: v1.ResourceRequirements{
Limits: limit,
},
Env: []v1.EnvVar{
{
Name: "POD_NAME",
@@ -220,6 +212,13 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string)
}
}
// specify resource limits if specified for the servers.
if s.cluster.Spec.ServerLimit != nil {
podSpec.Containers[0].Resources = v1.ResourceRequirements{
Limits: s.cluster.Spec.ServerLimit,
}
}
return podSpec
}
pkg/controller/clusterset/clusterset.go
+105 -24
View File
@@ -16,7 +16,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -32,7 +31,7 @@ const (
)
type ClusterSetReconciler struct {
Client ctrlruntimeclient.Client
Client client.Client
Scheme *runtime.Scheme
ClusterCIDR string
}
@@ -143,27 +142,14 @@ func (c *ClusterSetReconciler) Reconcile(ctx context.Context, req reconcile.Requ
return reconcile.Result{}, err
}
// TODO: Add resource quota for clustersets
// if clusterSet.Spec.MaxLimits != nil {
// quota := v1.ResourceQuota{
// ObjectMeta: metav1.ObjectMeta{
// Name: "clusterset-quota",
// Namespace: clusterSet.Namespace,
// OwnerReferences: []metav1.OwnerReference{
// {
// UID: clusterSet.UID,
// Name: clusterSet.Name,
// APIVersion: clusterSet.APIVersion,
// Kind: clusterSet.Kind,
// },
// },
// },
// }
// quota.Spec.Hard = clusterSet.Spec.MaxLimits
// if err := c.Client.Create(ctx, &quota); err != nil {
// return reconcile.Result{}, fmt.Errorf("unable to create resource quota from cluster set: %w", err)
// }
// }
if err := c.reconcileLimit(ctx, &clusterSet); err != nil {
return reconcile.Result{}, err
}
if err := c.reconcileQuota(ctx, &clusterSet); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
@@ -315,7 +301,7 @@ func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, clusterSet
log.Info("reconciling Clusters")
var clusters v1alpha1.ClusterList
if err := c.Client.List(ctx, &clusters, ctrlruntimeclient.InNamespace(clusterSet.Namespace)); err != nil {
if err := c.Client.List(ctx, &clusters, client.InNamespace(clusterSet.Namespace)); err != nil {
return err
}
@@ -340,3 +326,98 @@ func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, clusterSet
return err
}
func (c *ClusterSetReconciler) reconcileQuota(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error {
if clusterSet.Spec.Quota == nil {
// check if resourceQuota object exists and deletes it.
var toDeleteResourceQuota v1.ResourceQuota
key := types.NamespacedName{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: clusterSet.Namespace,
}
if err := c.Client.Get(ctx, key, &toDeleteResourceQuota); err != nil {
return client.IgnoreNotFound(err)
}
return c.Client.Delete(ctx, &toDeleteResourceQuota)
}
// create/update resource Quota
resourceQuota := resourceQuota(clusterSet)
if err := ctrl.SetControllerReference(clusterSet, &resourceQuota, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, &resourceQuota); err != nil {
if apierrors.IsAlreadyExists(err) {
return c.Client.Update(ctx, &resourceQuota)
}
}
return nil
}
func resourceQuota(clusterSet *v1alpha1.ClusterSet) v1.ResourceQuota {
return v1.ResourceQuota{
ObjectMeta: metav1.ObjectMeta{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: clusterSet.Namespace,
},
TypeMeta: metav1.TypeMeta{
Kind: "ResourceQuota",
APIVersion: "v1",
},
Spec: *clusterSet.Spec.Quota,
}
}
func (c *ClusterSetReconciler) reconcileLimit(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error {
log := ctrl.LoggerFrom(ctx)
log.Info("Reconciling ClusterSet Limit")
// delete limitrange if spec.limits isnt specified.
if clusterSet.Spec.Limit == nil {
var toDeleteLimitRange v1.LimitRange
key := types.NamespacedName{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: clusterSet.Namespace,
}
if err := c.Client.Get(ctx, key, &toDeleteLimitRange); err != nil {
return client.IgnoreNotFound(err)
}
return c.Client.Delete(ctx, &toDeleteLimitRange)
}
limitRange := limitRange(clusterSet)
if err := ctrl.SetControllerReference(clusterSet, &limitRange, c.Scheme); err != nil {
return err
}
if err := c.Client.Create(ctx, &limitRange); err != nil {
if apierrors.IsAlreadyExists(err) {
return c.Client.Update(ctx, &limitRange)
}
}
return nil
}
func limitRange(clusterSet *v1alpha1.ClusterSet) v1.LimitRange {
return v1.LimitRange{
ObjectMeta: metav1.ObjectMeta{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: clusterSet.Namespace,
},
TypeMeta: metav1.TypeMeta{
Kind: "LimitRange",
APIVersion: "v1",
},
Spec: *clusterSet.Spec.Limit,
}
}
pkg/controller/clusterset/clusterset_test.go
+151 -24
View File
@@ -8,11 +8,11 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
k3kcontroller "github.com/rancher/k3k/pkg/controller"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
@@ -29,7 +29,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
)
BeforeEach(func() {
createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
createdNS := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
err := k8sClient.Create(context.Background(), createdNS)
Expect(err).To(Not(HaveOccurred()))
namespace = createdNS.Name
@@ -38,7 +38,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
When("created with a default spec", func() {
It("should have only the 'shared' allowedModeTypes", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -54,7 +54,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
It("should create a NetworkPolicy", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -118,7 +118,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
When("created with DisableNetworkPolicy", func() {
It("should not create a NetworkPolicy if true", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -147,7 +147,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
It("should delete the NetworkPolicy if changed to false", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -191,7 +191,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
It("should recreate the NetworkPolicy if deleted", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -242,7 +242,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
When("created specifying the mode", func() {
It("should have the 'virtual' mode if specified", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -263,7 +263,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
It("should have both modes if specified", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -288,7 +288,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
It("should fail for a non-existing mode", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -315,7 +315,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
)
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -327,7 +327,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
err := k8sClient.Create(ctx, clusterSet)
Expect(err).To(Not(HaveOccurred()))
var ns corev1.Namespace
var ns v1.Namespace
// Check privileged
@@ -418,7 +418,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
privileged := v1alpha1.PrivilegedPodSecurityAdmissionLevel
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -430,7 +430,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
err := k8sClient.Create(ctx, clusterSet)
Expect(err).To(Not(HaveOccurred()))
var ns corev1.Namespace
var ns v1.Namespace
// wait a bit for the namespace to be updated
Eventually(func() bool {
@@ -469,7 +469,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
When("a cluster in the same namespace is present", func() {
It("should update it if needed", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -482,7 +482,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
Expect(err).To(Not(HaveOccurred()))
cluster := &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace,
},
@@ -510,7 +510,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
It("should update the nodeSelector", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -523,7 +523,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
Expect(err).To(Not(HaveOccurred()))
cluster := &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace,
},
@@ -551,7 +551,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
It("should update the nodeSelector if changed", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -564,7 +564,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
Expect(err).To(Not(HaveOccurred()))
cluster := &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace,
},
@@ -622,7 +622,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
When("a cluster in a different namespace is present", func() {
It("should not be update", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
@@ -634,12 +634,12 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
err := k8sClient.Create(ctx, clusterSet)
Expect(err).To(Not(HaveOccurred()))
namespace2 := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
namespace2 := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
err = k8sClient.Create(ctx, namespace2)
Expect(err).To(Not(HaveOccurred()))
cluster := &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "cluster-",
Namespace: namespace2.Name,
},
@@ -666,5 +666,132 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
Should(BeTrue())
})
})
When("created with ResourceQuota", func() {
It("should create resourceQuota if Quota is enabled", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
Spec: v1alpha1.ClusterSetSpec{
Quota: &v1.ResourceQuotaSpec{
Hard: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("800m"),
v1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
}
err := k8sClient.Create(ctx, clusterSet)
Expect(err).To(Not(HaveOccurred()))
var resourceQuota v1.ResourceQuota
Eventually(func() error {
key := types.NamespacedName{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: namespace,
}
return k8sClient.Get(ctx, key, &resourceQuota)
}).
WithTimeout(time.Second * 10).
WithPolling(time.Second).
Should(BeNil())
Expect(resourceQuota.Spec.Hard.Cpu().String()).To(BeEquivalentTo("800m"))
Expect(resourceQuota.Spec.Hard.Memory().String()).To(BeEquivalentTo("1Gi"))
})
It("should delete the ResourceQuota if Quota is deleted", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
Spec: v1alpha1.ClusterSetSpec{
Quota: &v1.ResourceQuotaSpec{
Hard: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("800m"),
v1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
}
err := k8sClient.Create(ctx, clusterSet)
Expect(err).To(Not(HaveOccurred()))
var resourceQuota v1.ResourceQuota
Eventually(func() error {
key := types.NamespacedName{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: namespace,
}
return k8sClient.Get(ctx, key, &resourceQuota)
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(BeNil())
clusterSet.Spec.Quota = nil
err = k8sClient.Update(ctx, clusterSet)
Expect(err).To(Not(HaveOccurred()))
// wait for a bit for the resourceQuota to be deleted
Eventually(func() bool {
key := types.NamespacedName{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: namespace,
}
err := k8sClient.Get(ctx, key, &resourceQuota)
return apierrors.IsNotFound(err)
}).
WithTimeout(time.Second * 10).
WithPolling(time.Second).
Should(BeTrue())
})
It("should create resourceQuota if Quota is enabled", func() {
clusterSet := &v1alpha1.ClusterSet{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "clusterset-",
Namespace: namespace,
},
Spec: v1alpha1.ClusterSetSpec{
Limit: &v1.LimitRangeSpec{
Limits: []v1.LimitRangeItem{
{
Type: v1.LimitTypeContainer,
DefaultRequest: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("500m"),
},
},
},
},
},
}
err := k8sClient.Create(ctx, clusterSet)
Expect(err).To(Not(HaveOccurred()))
var limitRange v1.LimitRange
Eventually(func() error {
key := types.NamespacedName{
Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
Namespace: namespace,
}
return k8sClient.Get(ctx, key, &limitRange)
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(BeNil())
// make sure that default limit range has the default requet values.
Expect(limitRange.Spec.Limits).ShouldNot(BeEmpty())
cpu := limitRange.Spec.Limits[0].DefaultRequest.Cpu().String()
Expect(cpu).To(BeEquivalentTo("500m"))
})
})
})
})