diff --git a/charts/k3k/crds/k3k.io_clusters.yaml b/charts/k3k/crds/k3k.io_clusters.yaml index af9ac579..d4630984 100644 --- a/charts/k3k/crds/k3k.io_clusters.yaml +++ b/charts/k3k/crds/k3k.io_clusters.yaml @@ -94,29 +94,6 @@ spec: x-kubernetes-validations: - message: clusterDNS is immutable rule: self == oldSelf - clusterLimit: - description: Limit defines resource limits for server/agent nodes. - properties: - serverLimit: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ServerLimit specifies resource limits for server - nodes. - type: object - workerLimit: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: WorkerLimit specifies resource limits for agent nodes. - type: object - type: object expose: description: |- Expose specifies options for exposing the API server. @@ -225,6 +202,15 @@ spec: items: type: string type: array + serverLimit: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ServerLimit specifies resource limits for server nodes. + type: object servers: default: 1 description: |- @@ -271,6 +257,15 @@ spec: It should follow the K3s versioning convention (e.g., v1.28.2-k3s1). If not specified, the Kubernetes version of the host node will be used. type: string + workerLimit: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: WorkerLimit specifies resource limits for agent nodes. + type: object type: object status: description: Status reflects the observed state of the Cluster. diff --git a/charts/k3k/crds/k3k.io_clustersets.yaml b/charts/k3k/crds/k3k.io_clustersets.yaml index e21e5b33..424c5476 100644 --- a/charts/k3k/crds/k3k.io_clustersets.yaml +++ b/charts/k3k/crds/k3k.io_clustersets.yaml @@ -59,30 +59,6 @@ spec: x-kubernetes-validations: - message: mode is immutable rule: self == oldSelf - defaultLimits: - description: DefaultLimits specifies the default resource limits for - servers/agents when a cluster in the set doesn't provide any. - properties: - serverLimit: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ServerLimit specifies resource limits for server - nodes. - type: object - workerLimit: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: WorkerLimit specifies resource limits for agent nodes. - type: object - type: object defaultNodeSelector: additionalProperties: type: string @@ -97,15 +73,81 @@ spec: description: DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. type: boolean - maxLimits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: MaxLimits specifies the maximum resource limits that - apply to all clusters (server + agent) in the set. + limit: + description: |- + Limit specifies the LimitRange that will be applied to all pods within the ClusterSet + to set defaults and constraints (min/max) + properties: + limits: + description: Limits is the list of LimitRangeItem objects that + are enforced. + items: + description: LimitRangeItem defines a min/max usage limit for + any resource that matches on kind. + properties: + default: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Default resource requirement limit value by + resource name if resource limit is omitted. + type: object + defaultRequest: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: DefaultRequest is the default resource requirement + request value by resource name if resource request is + omitted. + type: object + max: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Max usage constraints on this kind by resource + name. + type: object + maxLimitRequestRatio: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: MaxLimitRequestRatio if specified, the named + resource must have a request and limit that are both non-zero + where limit divided by request is less than or equal to + the enumerated value; this represents the max burst for + the named resource. + type: object + min: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Min usage constraints on this kind by resource + name. + type: object + type: + description: Type of resource that this limit applies to. + type: string + required: + - type + type: object + type: array + required: + - limits type: object podSecurityAdmissionLevel: description: PodSecurityAdmissionLevel specifies the pod security @@ -115,6 +157,70 @@ spec: - baseline - restricted type: string + quota: + description: Quota specifies the resource limits for clusters within + a clusterset. + properties: + hard: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + hard is the set of desired hard limits for each named resource. + More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/ + type: object + scopeSelector: + description: |- + scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota + but expressed using ScopeSelectorOperator in combination with possible values. + For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. + properties: + matchExpressions: + description: A list of scope selector requirements by scope + of the resources. + items: + description: |- + A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator + that relates the scope name and values. + properties: + operator: + description: |- + Represents a scope's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. + type: string + scopeName: + description: The name of the scope that the selector + applies to. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - operator + - scopeName + type: object + type: array + type: object + x-kubernetes-map-type: atomic + scopes: + description: |- + A collection of filters that must match each object tracked by a quota. + If not specified, the quota matches all objects. + items: + description: A ResourceQuotaScope defines a filter that must + match each object tracked by a quota + type: string + type: array + type: object type: object status: description: Status reflects the observed state of the ClusterSet. diff --git a/docs/crds/crd-docs.md b/docs/crds/crd-docs.md index 6f7d304d..72bc3708 100644 --- a/docs/crds/crd-docs.md +++ b/docs/crds/crd-docs.md @@ -51,23 +51,6 @@ _Appears in:_ | `spec` _[ClusterSpec](#clusterspec)_ | Spec defines the desired state of the Cluster. | \{ \} | | -#### ClusterLimit - - - -ClusterLimit defines resource limits for server and agent nodes. - - - -_Appears in:_ -- [ClusterSpec](#clusterspec) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit specifies resource limits for server nodes. | | | -| `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit specifies resource limits for agent nodes. | | | - - #### ClusterList @@ -124,12 +107,13 @@ _Appears in:_ | `expose` _[ExposeConfig](#exposeconfig)_ | Expose specifies options for exposing the API server.
By default, it's only exposed as a ClusterIP. | | | | `nodeSelector` _object (keys:string, values:string)_ | NodeSelector specifies node labels to constrain where server/agent pods are scheduled.
In "shared" mode, this also applies to workloads. | | | | `priorityClass` _string_ | PriorityClass specifies the priorityClassName for server/agent pods.
In "shared" mode, this also applies to workloads. | | | -| `clusterLimit` _[ClusterLimit](#clusterlimit)_ | Limit defines resource limits for server/agent nodes. | | | | `tokenSecretRef` _[SecretReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#secretreference-v1-core)_ | TokenSecretRef is a Secret reference containing the token used by worker nodes to join the cluster.
The Secret must have a "token" field in its data. | | | | `tlsSANs` _string array_ | TLSSANs specifies subject alternative names for the K3s server certificate. | | | | `serverArgs` _string array_ | ServerArgs specifies ordered key-value pairs for K3s server pods.
Example: ["--tls-san=example.com"] | | | | `agentArgs` _string array_ | AgentArgs specifies ordered key-value pairs for K3s agent pods.
Example: ["--node-name=my-agent-node"] | | | | `addons` _[Addon](#addon) array_ | Addons specifies secrets containing raw YAML to deploy on cluster startup. | | | +| `serverLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | ServerLimit specifies resource limits for server nodes. | | | +| `workerLimit` _[ResourceList](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#resourcelist-v1-core)_ | WorkerLimit specifies resource limits for agent nodes. | | | diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index e22a069b..16f785c1 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -114,11 +114,6 @@ type ClusterSpec struct { // +optional PriorityClass string `json:"priorityClass,omitempty"` - // Limit defines resource limits for server/agent nodes. - // - // +optional - Limit *ClusterLimit `json:"clusterLimit,omitempty"` - // TokenSecretRef is a Secret reference containing the token used by worker nodes to join the cluster. // The Secret must have a "token" field in its data. // @@ -146,6 +141,16 @@ type ClusterSpec struct { // // +optional Addons []Addon `json:"addons,omitempty"` + + // ServerLimit specifies resource limits for server nodes. + // + // +optional + ServerLimit v1.ResourceList `json:"serverLimit,omitempty"` + + // WorkerLimit specifies resource limits for agent nodes. + // + // +optional + WorkerLimit v1.ResourceList `json:"workerLimit,omitempty"` } // ClusterMode is the possible provisioning mode of a Cluster. @@ -175,15 +180,6 @@ const ( DynamicPersistenceMode = PersistenceMode("dynamic") ) -// ClusterLimit defines resource limits for server and agent nodes. -type ClusterLimit struct { - // ServerLimit specifies resource limits for server nodes. - ServerLimit v1.ResourceList `json:"serverLimit,omitempty"` - - // WorkerLimit specifies resource limits for agent nodes. - WorkerLimit v1.ResourceList `json:"workerLimit,omitempty"` -} - // Addon specifies a Secret containing YAML to be deployed on cluster startup. type Addon struct { // SecretNamespace is the namespace of the Secret. @@ -338,10 +334,16 @@ type ClusterSet struct { // ClusterSetSpec defines the desired state of a ClusterSet. type ClusterSetSpec struct { - // DefaultLimits specifies the default resource limits for servers/agents when a cluster in the set doesn't provide any. + // Quota specifies the resource limits for clusters within a clusterset. // // +optional - DefaultLimits *ClusterLimit `json:"defaultLimits,omitempty"` + Quota *v1.ResourceQuotaSpec `json:"quota,omitempty"` + + // Limit specifies the LimitRange that will be applied to all pods within the ClusterSet + // to set defaults and constraints (min/max) + // + // +optional + Limit *v1.LimitRangeSpec `json:"limit,omitempty"` // DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the set. // @@ -353,11 +355,6 @@ type ClusterSetSpec struct { // +optional DefaultPriorityClass string `json:"defaultPriorityClass,omitempty"` - // MaxLimits specifies the maximum resource limits that apply to all clusters (server + agent) in the set. - // - // +optional - MaxLimits v1.ResourceList `json:"maxLimits,omitempty"` - // AllowedModeTypes specifies the allowed cluster provisioning modes. Defaults to [shared]. // // +kubebuilder:default={shared} diff --git a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go index 9366f37e..d85f3866 100644 --- a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go @@ -55,36 +55,6 @@ func (in *Cluster) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterLimit) DeepCopyInto(out *ClusterLimit) { - *out = *in - if in.ServerLimit != nil { - in, out := &in.ServerLimit, &out.ServerLimit - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - if in.WorkerLimit != nil { - in, out := &in.WorkerLimit, &out.WorkerLimit - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLimit. -func (in *ClusterLimit) DeepCopy() *ClusterLimit { - if in == nil { - return nil - } - out := new(ClusterLimit) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterList) DeepCopyInto(out *ClusterList) { *out = *in @@ -182,16 +152,14 @@ func (in *ClusterSetList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterSetSpec) DeepCopyInto(out *ClusterSetSpec) { *out = *in - if in.MaxLimits != nil { - in, out := &in.MaxLimits, &out.MaxLimits - *out = make(v1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } + if in.Quota != nil { + in, out := &in.Quota, &out.Quota + *out = new(v1.ResourceQuotaSpec) + (*in).DeepCopyInto(*out) } - if in.DefaultLimits != nil { - in, out := &in.DefaultLimits, &out.DefaultLimits - *out = new(ClusterLimit) + if in.Limit != nil { + in, out := &in.Limit, &out.Limit + *out = new(v1.LimitRangeSpec) (*in).DeepCopyInto(*out) } if in.DefaultNodeSelector != nil { @@ -260,6 +228,12 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(int32) **out = **in } + in.Persistence.DeepCopyInto(&out.Persistence) + if in.Expose != nil { + in, out := &in.Expose, &out.Expose + *out = new(ExposeConfig) + (*in).DeepCopyInto(*out) + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) @@ -267,16 +241,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { (*out)[key] = val } } - if in.Limit != nil { - in, out := &in.Limit, &out.Limit - *out = new(ClusterLimit) - (*in).DeepCopyInto(*out) - } if in.TokenSecretRef != nil { in, out := &in.TokenSecretRef, &out.TokenSecretRef *out = new(v1.SecretReference) **out = **in } + if in.TLSSANs != nil { + in, out := &in.TLSSANs, &out.TLSSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.ServerArgs != nil { in, out := &in.ServerArgs, &out.ServerArgs *out = make([]string, len(*in)) @@ -287,21 +261,24 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.TLSSANs != nil { - in, out := &in.TLSSANs, &out.TLSSANs - *out = make([]string, len(*in)) - copy(*out, *in) - } if in.Addons != nil { in, out := &in.Addons, &out.Addons *out = make([]Addon, len(*in)) copy(*out, *in) } - in.Persistence.DeepCopyInto(&out.Persistence) - if in.Expose != nil { - in, out := &in.Expose, &out.Expose - *out = new(ExposeConfig) - (*in).DeepCopyInto(*out) + if in.ServerLimit != nil { + in, out := &in.ServerLimit, &out.ServerLimit + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.WorkerLimit != nil { + in, out := &in.WorkerLimit, &out.WorkerLimit + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } } return } diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index 3f1fb394..9c0cdc3d 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -228,5 +228,12 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect }, } + // specify resource limits if specified for the servers. + if v.cluster.Spec.WorkerLimit != nil { + podSpec.Containers[0].Resources = v1.ResourceRequirements{ + Limits: v.cluster.Spec.WorkerLimit, + } + } + return podSpec } diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 3b994890..9dc66458 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -46,11 +46,6 @@ func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string) *S } func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) v1.PodSpec { - var limit v1.ResourceList - if s.cluster.Spec.Limit != nil && s.cluster.Spec.Limit.ServerLimit != nil { - limit = s.cluster.Spec.Limit.ServerLimit - } - podSpec := v1.PodSpec{ NodeSelector: s.cluster.Spec.NodeSelector, PriorityClassName: s.cluster.Spec.PriorityClass, @@ -118,9 +113,6 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) { Name: name, Image: image, - Resources: v1.ResourceRequirements{ - Limits: limit, - }, Env: []v1.EnvVar{ { Name: "POD_NAME", @@ -220,6 +212,13 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) } } + // specify resource limits if specified for the servers. + if s.cluster.Spec.ServerLimit != nil { + podSpec.Containers[0].Resources = v1.ResourceRequirements{ + Limits: s.cluster.Spec.ServerLimit, + } + } + return podSpec } diff --git a/pkg/controller/clusterset/clusterset.go b/pkg/controller/clusterset/clusterset.go index 5316d56a..b0b5818d 100644 --- a/pkg/controller/clusterset/clusterset.go +++ b/pkg/controller/clusterset/clusterset.go @@ -16,7 +16,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -32,7 +31,7 @@ const ( ) type ClusterSetReconciler struct { - Client ctrlruntimeclient.Client + Client client.Client Scheme *runtime.Scheme ClusterCIDR string } @@ -143,27 +142,14 @@ func (c *ClusterSetReconciler) Reconcile(ctx context.Context, req reconcile.Requ return reconcile.Result{}, err } - // TODO: Add resource quota for clustersets - // if clusterSet.Spec.MaxLimits != nil { - // quota := v1.ResourceQuota{ - // ObjectMeta: metav1.ObjectMeta{ - // Name: "clusterset-quota", - // Namespace: clusterSet.Namespace, - // OwnerReferences: []metav1.OwnerReference{ - // { - // UID: clusterSet.UID, - // Name: clusterSet.Name, - // APIVersion: clusterSet.APIVersion, - // Kind: clusterSet.Kind, - // }, - // }, - // }, - // } - // quota.Spec.Hard = clusterSet.Spec.MaxLimits - // if err := c.Client.Create(ctx, "a); err != nil { - // return reconcile.Result{}, fmt.Errorf("unable to create resource quota from cluster set: %w", err) - // } - // } + if err := c.reconcileLimit(ctx, &clusterSet); err != nil { + return reconcile.Result{}, err + } + + if err := c.reconcileQuota(ctx, &clusterSet); err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{}, nil } @@ -315,7 +301,7 @@ func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, clusterSet log.Info("reconciling Clusters") var clusters v1alpha1.ClusterList - if err := c.Client.List(ctx, &clusters, ctrlruntimeclient.InNamespace(clusterSet.Namespace)); err != nil { + if err := c.Client.List(ctx, &clusters, client.InNamespace(clusterSet.Namespace)); err != nil { return err } @@ -340,3 +326,98 @@ func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, clusterSet return err } + +func (c *ClusterSetReconciler) reconcileQuota(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error { + if clusterSet.Spec.Quota == nil { + // check if resourceQuota object exists and deletes it. + var toDeleteResourceQuota v1.ResourceQuota + + key := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: clusterSet.Namespace, + } + + if err := c.Client.Get(ctx, key, &toDeleteResourceQuota); err != nil { + return client.IgnoreNotFound(err) + } + + return c.Client.Delete(ctx, &toDeleteResourceQuota) + } + + // create/update resource Quota + resourceQuota := resourceQuota(clusterSet) + + if err := ctrl.SetControllerReference(clusterSet, &resourceQuota, c.Scheme); err != nil { + return err + } + + if err := c.Client.Create(ctx, &resourceQuota); err != nil { + if apierrors.IsAlreadyExists(err) { + return c.Client.Update(ctx, &resourceQuota) + } + } + + return nil +} + +func resourceQuota(clusterSet *v1alpha1.ClusterSet) v1.ResourceQuota { + return v1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: clusterSet.Namespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "ResourceQuota", + APIVersion: "v1", + }, + Spec: *clusterSet.Spec.Quota, + } +} + +func (c *ClusterSetReconciler) reconcileLimit(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error { + log := ctrl.LoggerFrom(ctx) + log.Info("Reconciling ClusterSet Limit") + + // delete limitrange if spec.limits isnt specified. + if clusterSet.Spec.Limit == nil { + var toDeleteLimitRange v1.LimitRange + + key := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: clusterSet.Namespace, + } + + if err := c.Client.Get(ctx, key, &toDeleteLimitRange); err != nil { + return client.IgnoreNotFound(err) + } + + return c.Client.Delete(ctx, &toDeleteLimitRange) + } + + limitRange := limitRange(clusterSet) + if err := ctrl.SetControllerReference(clusterSet, &limitRange, c.Scheme); err != nil { + return err + } + + if err := c.Client.Create(ctx, &limitRange); err != nil { + if apierrors.IsAlreadyExists(err) { + return c.Client.Update(ctx, &limitRange) + } + } + + return nil +} + +func limitRange(clusterSet *v1alpha1.ClusterSet) v1.LimitRange { + return v1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: clusterSet.Namespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "LimitRange", + APIVersion: "v1", + }, + Spec: *clusterSet.Spec.Limit, + } +} diff --git a/pkg/controller/clusterset/clusterset_test.go b/pkg/controller/clusterset/clusterset_test.go index 750bfcc6..9d97fe8b 100644 --- a/pkg/controller/clusterset/clusterset_test.go +++ b/pkg/controller/clusterset/clusterset_test.go @@ -8,11 +8,11 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" k3kcontroller "github.com/rancher/k3k/pkg/controller" - corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -29,7 +29,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet ) BeforeEach(func() { - createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}} + createdNS := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}} err := k8sClient.Create(context.Background(), createdNS) Expect(err).To(Not(HaveOccurred())) namespace = createdNS.Name @@ -38,7 +38,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet When("created with a default spec", func() { It("should have only the 'shared' allowedModeTypes", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -54,7 +54,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet It("should create a NetworkPolicy", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -118,7 +118,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet When("created with DisableNetworkPolicy", func() { It("should not create a NetworkPolicy if true", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -147,7 +147,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet It("should delete the NetworkPolicy if changed to false", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -191,7 +191,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet It("should recreate the NetworkPolicy if deleted", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -242,7 +242,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet When("created specifying the mode", func() { It("should have the 'virtual' mode if specified", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -263,7 +263,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet It("should have both modes if specified", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -288,7 +288,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet It("should fail for a non-existing mode", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -315,7 +315,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet ) clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -327,7 +327,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet err := k8sClient.Create(ctx, clusterSet) Expect(err).To(Not(HaveOccurred())) - var ns corev1.Namespace + var ns v1.Namespace // Check privileged @@ -418,7 +418,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet privileged := v1alpha1.PrivilegedPodSecurityAdmissionLevel clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -430,7 +430,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet err := k8sClient.Create(ctx, clusterSet) Expect(err).To(Not(HaveOccurred())) - var ns corev1.Namespace + var ns v1.Namespace // wait a bit for the namespace to be updated Eventually(func() bool { @@ -469,7 +469,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet When("a cluster in the same namespace is present", func() { It("should update it if needed", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -482,7 +482,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet Expect(err).To(Not(HaveOccurred())) cluster := &v1alpha1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "cluster-", Namespace: namespace, }, @@ -510,7 +510,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet It("should update the nodeSelector", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -523,7 +523,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet Expect(err).To(Not(HaveOccurred())) cluster := &v1alpha1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "cluster-", Namespace: namespace, }, @@ -551,7 +551,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet It("should update the nodeSelector if changed", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -564,7 +564,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet Expect(err).To(Not(HaveOccurred())) cluster := &v1alpha1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "cluster-", Namespace: namespace, }, @@ -622,7 +622,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet When("a cluster in a different namespace is present", func() { It("should not be update", func() { clusterSet := &v1alpha1.ClusterSet{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "clusterset-", Namespace: namespace, }, @@ -634,12 +634,12 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet err := k8sClient.Create(ctx, clusterSet) Expect(err).To(Not(HaveOccurred())) - namespace2 := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}} + namespace2 := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}} err = k8sClient.Create(ctx, namespace2) Expect(err).To(Not(HaveOccurred())) cluster := &v1alpha1.Cluster{ - ObjectMeta: v1.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ GenerateName: "cluster-", Namespace: namespace2.Name, }, @@ -666,5 +666,132 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet Should(BeTrue()) }) }) + + When("created with ResourceQuota", func() { + It("should create resourceQuota if Quota is enabled", func() { + clusterSet := &v1alpha1.ClusterSet{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "clusterset-", + Namespace: namespace, + }, + Spec: v1alpha1.ClusterSetSpec{ + Quota: &v1.ResourceQuotaSpec{ + Hard: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("800m"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, + } + + err := k8sClient.Create(ctx, clusterSet) + Expect(err).To(Not(HaveOccurred())) + + var resourceQuota v1.ResourceQuota + Eventually(func() error { + key := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: namespace, + } + + return k8sClient.Get(ctx, key, &resourceQuota) + }). + WithTimeout(time.Second * 10). + WithPolling(time.Second). + Should(BeNil()) + Expect(resourceQuota.Spec.Hard.Cpu().String()).To(BeEquivalentTo("800m")) + Expect(resourceQuota.Spec.Hard.Memory().String()).To(BeEquivalentTo("1Gi")) + }) + It("should delete the ResourceQuota if Quota is deleted", func() { + clusterSet := &v1alpha1.ClusterSet{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "clusterset-", + Namespace: namespace, + }, + Spec: v1alpha1.ClusterSetSpec{ + Quota: &v1.ResourceQuotaSpec{ + Hard: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("800m"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + }, + } + + err := k8sClient.Create(ctx, clusterSet) + Expect(err).To(Not(HaveOccurred())) + + var resourceQuota v1.ResourceQuota + + Eventually(func() error { + key := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: namespace, + } + return k8sClient.Get(ctx, key, &resourceQuota) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(BeNil()) + + clusterSet.Spec.Quota = nil + err = k8sClient.Update(ctx, clusterSet) + Expect(err).To(Not(HaveOccurred())) + + // wait for a bit for the resourceQuota to be deleted + Eventually(func() bool { + key := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: namespace, + } + err := k8sClient.Get(ctx, key, &resourceQuota) + return apierrors.IsNotFound(err) + }). + WithTimeout(time.Second * 10). + WithPolling(time.Second). + Should(BeTrue()) + }) + It("should create resourceQuota if Quota is enabled", func() { + clusterSet := &v1alpha1.ClusterSet{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "clusterset-", + Namespace: namespace, + }, + Spec: v1alpha1.ClusterSetSpec{ + Limit: &v1.LimitRangeSpec{ + Limits: []v1.LimitRangeItem{ + { + Type: v1.LimitTypeContainer, + DefaultRequest: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("500m"), + }, + }, + }, + }, + }, + } + + err := k8sClient.Create(ctx, clusterSet) + Expect(err).To(Not(HaveOccurred())) + + var limitRange v1.LimitRange + + Eventually(func() error { + key := types.NamespacedName{ + Name: k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name), + Namespace: namespace, + } + return k8sClient.Get(ctx, key, &limitRange) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(BeNil()) + + // make sure that default limit range has the default requet values. + Expect(limitRange.Spec.Limits).ShouldNot(BeEmpty()) + cpu := limitRange.Spec.Limits[0].DefaultRequest.Cpu().String() + Expect(cpu).To(BeEquivalentTo("500m")) + }) + }) }) })