Initial support for ResourceQuotas in clustersets (#308)

* Add ResourceQuota to clusterset Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl fixes Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Generate docs Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix tests Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Add a defualt limitRange for ClusterSets Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix linting Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix docs Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Add test for clusterset limitRange Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Add server and worker limits Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * make charts Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * add default limits and fixes to resourcesquota Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl lint Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl lint Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * make docs Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * make build-crds Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * make build-crds Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * make spec as pointer Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix tests Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix tests Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * delete default limit Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl lint Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Update tests Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Update tests Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * return on delete in limitrange Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * wsl Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> --------- Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
2026-08-18 20:07:06 +00:00 · 2025-04-03 12:30:48 +02:00
parent b58578788c
commit 0843a9e313
9 changed files with 478 additions and 205 deletions
@@ -228,5 +228,12 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect
 		},
 	}

+	// specify resource limits if specified for the servers.
+	if v.cluster.Spec.WorkerLimit != nil {
+		podSpec.Containers[0].Resources = v1.ResourceRequirements{
+			Limits: v.cluster.Spec.WorkerLimit,
+		}
+	}
+
 	return podSpec
 }
@@ -46,11 +46,6 @@ func New(cluster *v1alpha1.Cluster, client client.Client, token, mode string) *S
 }

 func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) v1.PodSpec {
-	var limit v1.ResourceList
-	if s.cluster.Spec.Limit != nil && s.cluster.Spec.Limit.ServerLimit != nil {
-		limit = s.cluster.Spec.Limit.ServerLimit
-	}
-
 	podSpec := v1.PodSpec{
 		NodeSelector:      s.cluster.Spec.NodeSelector,
 		PriorityClassName: s.cluster.Spec.PriorityClass,
@@ -118,9 +113,6 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string)
 			{
 				Name:  name,
 				Image: image,
-				Resources: v1.ResourceRequirements{
-					Limits: limit,
-				},
 				Env: []v1.EnvVar{
 					{
 						Name: "POD_NAME",
@@ -220,6 +212,13 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string)
 		}
 	}

+	// specify resource limits if specified for the servers.
+	if s.cluster.Spec.ServerLimit != nil {
+		podSpec.Containers[0].Resources = v1.ResourceRequirements{
+			Limits: s.cluster.Spec.ServerLimit,
+		}
+	}
+
 	return podSpec
 }

@@ -16,7 +16,6 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -32,7 +31,7 @@ const (
 )

 type ClusterSetReconciler struct {
-	Client      ctrlruntimeclient.Client
+	Client      client.Client
 	Scheme      *runtime.Scheme
 	ClusterCIDR string
 }
@@ -143,27 +142,14 @@ func (c *ClusterSetReconciler) Reconcile(ctx context.Context, req reconcile.Requ
 		return reconcile.Result{}, err
 	}

-	// TODO: Add resource quota for clustersets
-	// if clusterSet.Spec.MaxLimits != nil {
-	// 	quota := v1.ResourceQuota{
-	// 		ObjectMeta: metav1.ObjectMeta{
-	// 			Name:      "clusterset-quota",
-	// 			Namespace: clusterSet.Namespace,
-	// 			OwnerReferences: []metav1.OwnerReference{
-	// 				{
-	// 					UID:        clusterSet.UID,
-	// 					Name:       clusterSet.Name,
-	// 					APIVersion: clusterSet.APIVersion,
-	// 					Kind:       clusterSet.Kind,
-	// 				},
-	// 			},
-	// 		},
-	// 	}
-	// 	quota.Spec.Hard = clusterSet.Spec.MaxLimits
-	// 	if err := c.Client.Create(ctx, &quota); err != nil {
-	// 		return reconcile.Result{}, fmt.Errorf("unable to create resource quota from cluster set: %w", err)
-	// 	}
-	// }
+	if err := c.reconcileLimit(ctx, &clusterSet); err != nil {
+		return reconcile.Result{}, err
+	}
+
+	if err := c.reconcileQuota(ctx, &clusterSet); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	return reconcile.Result{}, nil
 }

@@ -315,7 +301,7 @@ func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, clusterSet
 	log.Info("reconciling Clusters")

 	var clusters v1alpha1.ClusterList
-	if err := c.Client.List(ctx, &clusters, ctrlruntimeclient.InNamespace(clusterSet.Namespace)); err != nil {
+	if err := c.Client.List(ctx, &clusters, client.InNamespace(clusterSet.Namespace)); err != nil {
 		return err
 	}

@@ -340,3 +326,98 @@ func (c *ClusterSetReconciler) reconcileClusters(ctx context.Context, clusterSet

 	return err
 }
+
+func (c *ClusterSetReconciler) reconcileQuota(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error {
+	if clusterSet.Spec.Quota == nil {
+		// check if resourceQuota object exists and deletes it.
+		var toDeleteResourceQuota v1.ResourceQuota
+
+		key := types.NamespacedName{
+			Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+			Namespace: clusterSet.Namespace,
+		}
+
+		if err := c.Client.Get(ctx, key, &toDeleteResourceQuota); err != nil {
+			return client.IgnoreNotFound(err)
+		}
+
+		return c.Client.Delete(ctx, &toDeleteResourceQuota)
+	}
+
+	// create/update resource Quota
+	resourceQuota := resourceQuota(clusterSet)
+
+	if err := ctrl.SetControllerReference(clusterSet, &resourceQuota, c.Scheme); err != nil {
+		return err
+	}
+
+	if err := c.Client.Create(ctx, &resourceQuota); err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			return c.Client.Update(ctx, &resourceQuota)
+		}
+	}
+
+	return nil
+}
+
+func resourceQuota(clusterSet *v1alpha1.ClusterSet) v1.ResourceQuota {
+	return v1.ResourceQuota{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+			Namespace: clusterSet.Namespace,
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ResourceQuota",
+			APIVersion: "v1",
+		},
+		Spec: *clusterSet.Spec.Quota,
+	}
+}
+
+func (c *ClusterSetReconciler) reconcileLimit(ctx context.Context, clusterSet *v1alpha1.ClusterSet) error {
+	log := ctrl.LoggerFrom(ctx)
+	log.Info("Reconciling ClusterSet Limit")
+
+	// delete limitrange if spec.limits isnt specified.
+	if clusterSet.Spec.Limit == nil {
+		var toDeleteLimitRange v1.LimitRange
+
+		key := types.NamespacedName{
+			Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+			Namespace: clusterSet.Namespace,
+		}
+
+		if err := c.Client.Get(ctx, key, &toDeleteLimitRange); err != nil {
+			return client.IgnoreNotFound(err)
+		}
+
+		return c.Client.Delete(ctx, &toDeleteLimitRange)
+	}
+
+	limitRange := limitRange(clusterSet)
+	if err := ctrl.SetControllerReference(clusterSet, &limitRange, c.Scheme); err != nil {
+		return err
+	}
+
+	if err := c.Client.Create(ctx, &limitRange); err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			return c.Client.Update(ctx, &limitRange)
+		}
+	}
+
+	return nil
+}
+
+func limitRange(clusterSet *v1alpha1.ClusterSet) v1.LimitRange {
+	return v1.LimitRange{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+			Namespace: clusterSet.Namespace,
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "LimitRange",
+			APIVersion: "v1",
+		},
+		Spec: *clusterSet.Spec.Limit,
+	}
+}
@@ -8,11 +8,11 @@ import (
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"

 	k3kcontroller "github.com/rancher/k3k/pkg/controller"
-	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"

@@ -29,7 +29,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 		)

 		BeforeEach(func() {
-			createdNS := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+			createdNS := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
 			err := k8sClient.Create(context.Background(), createdNS)
 			Expect(err).To(Not(HaveOccurred()))
 			namespace = createdNS.Name
@@ -38,7 +38,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 		When("created with a default spec", func() {
 			It("should have only the 'shared' allowedModeTypes", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -54,7 +54,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet

 			It("should create a NetworkPolicy", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -118,7 +118,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 		When("created with DisableNetworkPolicy", func() {
 			It("should not create a NetworkPolicy if true", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -147,7 +147,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet

 			It("should delete the NetworkPolicy if changed to false", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -191,7 +191,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet

 			It("should recreate the NetworkPolicy if deleted", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -242,7 +242,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 		When("created specifying the mode", func() {
 			It("should have the 'virtual' mode if specified", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -263,7 +263,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet

 			It("should have both modes if specified", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -288,7 +288,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet

 			It("should fail for a non-existing mode", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -315,7 +315,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				)

 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -327,7 +327,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				err := k8sClient.Create(ctx, clusterSet)
 				Expect(err).To(Not(HaveOccurred()))

-				var ns corev1.Namespace
+				var ns v1.Namespace

 				// Check privileged

@@ -418,7 +418,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				privileged := v1alpha1.PrivilegedPodSecurityAdmissionLevel

 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -430,7 +430,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				err := k8sClient.Create(ctx, clusterSet)
 				Expect(err).To(Not(HaveOccurred()))

-				var ns corev1.Namespace
+				var ns v1.Namespace

 				// wait a bit for the namespace to be updated
 				Eventually(func() bool {
@@ -469,7 +469,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 		When("a cluster in the same namespace is present", func() {
 			It("should update it if needed", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -482,7 +482,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				Expect(err).To(Not(HaveOccurred()))

 				cluster := &v1alpha1.Cluster{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "cluster-",
 						Namespace:    namespace,
 					},
@@ -510,7 +510,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet

 			It("should update the nodeSelector", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -523,7 +523,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				Expect(err).To(Not(HaveOccurred()))

 				cluster := &v1alpha1.Cluster{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "cluster-",
 						Namespace:    namespace,
 					},
@@ -551,7 +551,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet

 			It("should update the nodeSelector if changed", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -564,7 +564,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				Expect(err).To(Not(HaveOccurred()))

 				cluster := &v1alpha1.Cluster{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "cluster-",
 						Namespace:    namespace,
 					},
@@ -622,7 +622,7 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 		When("a cluster in a different namespace is present", func() {
 			It("should not be update", func() {
 				clusterSet := &v1alpha1.ClusterSet{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "clusterset-",
 						Namespace:    namespace,
 					},
@@ -634,12 +634,12 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 				err := k8sClient.Create(ctx, clusterSet)
 				Expect(err).To(Not(HaveOccurred()))

-				namespace2 := &corev1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "ns-"}}
+				namespace2 := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
 				err = k8sClient.Create(ctx, namespace2)
 				Expect(err).To(Not(HaveOccurred()))

 				cluster := &v1alpha1.Cluster{
-					ObjectMeta: v1.ObjectMeta{
+					ObjectMeta: metav1.ObjectMeta{
 						GenerateName: "cluster-",
 						Namespace:    namespace2.Name,
 					},
@@ -666,5 +666,132 @@ var _ = Describe("ClusterSet Controller", Label("controller"), Label("ClusterSet
 					Should(BeTrue())
 			})
 		})
+
+		When("created with ResourceQuota", func() {
+			It("should create resourceQuota if Quota is enabled", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: metav1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						Quota: &v1.ResourceQuotaSpec{
+							Hard: v1.ResourceList{
+								v1.ResourceCPU:    resource.MustParse("800m"),
+								v1.ResourceMemory: resource.MustParse("1Gi"),
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				var resourceQuota v1.ResourceQuota
+				Eventually(func() error {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+
+					return k8sClient.Get(ctx, key, &resourceQuota)
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeNil())
+				Expect(resourceQuota.Spec.Hard.Cpu().String()).To(BeEquivalentTo("800m"))
+				Expect(resourceQuota.Spec.Hard.Memory().String()).To(BeEquivalentTo("1Gi"))
+			})
+			It("should delete the ResourceQuota if Quota is deleted", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: metav1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						Quota: &v1.ResourceQuotaSpec{
+							Hard: v1.ResourceList{
+								v1.ResourceCPU:    resource.MustParse("800m"),
+								v1.ResourceMemory: resource.MustParse("1Gi"),
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				var resourceQuota v1.ResourceQuota
+
+				Eventually(func() error {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					return k8sClient.Get(ctx, key, &resourceQuota)
+				}).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeNil())
+
+				clusterSet.Spec.Quota = nil
+				err = k8sClient.Update(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// wait for a bit for the resourceQuota to be deleted
+				Eventually(func() bool {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					err := k8sClient.Get(ctx, key, &resourceQuota)
+					return apierrors.IsNotFound(err)
+				}).
+					WithTimeout(time.Second * 10).
+					WithPolling(time.Second).
+					Should(BeTrue())
+			})
+			It("should create resourceQuota if Quota is enabled", func() {
+				clusterSet := &v1alpha1.ClusterSet{
+					ObjectMeta: metav1.ObjectMeta{
+						GenerateName: "clusterset-",
+						Namespace:    namespace,
+					},
+					Spec: v1alpha1.ClusterSetSpec{
+						Limit: &v1.LimitRangeSpec{
+							Limits: []v1.LimitRangeItem{
+								{
+									Type: v1.LimitTypeContainer,
+									DefaultRequest: v1.ResourceList{
+										v1.ResourceCPU: resource.MustParse("500m"),
+									},
+								},
+							},
+						},
+					},
+				}
+
+				err := k8sClient.Create(ctx, clusterSet)
+				Expect(err).To(Not(HaveOccurred()))
+
+				var limitRange v1.LimitRange
+
+				Eventually(func() error {
+					key := types.NamespacedName{
+						Name:      k3kcontroller.SafeConcatNameWithPrefix(clusterSet.Name),
+						Namespace: namespace,
+					}
+					return k8sClient.Get(ctx, key, &limitRange)
+				}).
+					WithTimeout(time.Minute).
+					WithPolling(time.Second).
+					Should(BeNil())
+
+				// make sure that default limit range has the default requet values.
+				Expect(limitRange.Spec.Limits).ShouldNot(BeEmpty())
+				cpu := limitRange.Spec.Limits[0].DefaultRequest.Cpu().String()
+				Expect(cpu).To(BeEquivalentTo("500m"))
+			})
+		})
 	})
 })