From 95e8a8c3129d5d089883a08202b3f22917452172 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:24:49 +0000 Subject: [PATCH] Add tenant node status collection and node watchers Agent-Logs-Url: https://github.com/projectcapsule/capsule/sessions/35d75f80-a2f5-4f3f-a0f8-db8fcf20edce Co-authored-by: oliverbaehler <26610571+oliverbaehler@users.noreply.github.com> --- api/v1beta1/tenant_status.go | 2 + api/v1beta2/tenant_conversion_hub.go | 2 + api/v1beta2/tenant_status.go | 3 + e2e/tenant_node_status_test.go | 203 +++++++++++++++++++++++++ internal/controllers/tenant/manager.go | 4 + internal/controllers/tenant/status.go | 38 +++++ internal/controllers/tenant/utils.go | 77 ++++++++++ 7 files changed, 329 insertions(+) create mode 100644 e2e/tenant_node_status_test.go diff --git a/api/v1beta1/tenant_status.go b/api/v1beta1/tenant_status.go index b1022c33..1695e1d9 100644 --- a/api/v1beta1/tenant_status.go +++ b/api/v1beta1/tenant_status.go @@ -13,6 +13,8 @@ const ( // Returns the observed state of the Tenant. type TenantStatus struct { + // List of nodes assigned to the Tenant. + Nodes []string `json:"nodes,omitempty"` // +kubebuilder:default=Active // The operational state of the Tenant. Possible values are "Active", "Cordoned". State tenantState `json:"state"` diff --git a/api/v1beta2/tenant_conversion_hub.go b/api/v1beta2/tenant_conversion_hub.go index dc12dcb6..dba02414 100644 --- a/api/v1beta2/tenant_conversion_hub.go +++ b/api/v1beta2/tenant_conversion_hub.go @@ -158,6 +158,7 @@ func (in *Tenant) ConvertFrom(raw conversion.Hub) error { in.SetAnnotations(annotations) in.Status.Namespaces = src.Status.Namespaces + in.Status.Nodes = src.Status.Nodes in.Status.Size = src.Status.Size switch src.Status.State { @@ -280,6 +281,7 @@ func (in *Tenant) ConvertTo(raw conversion.Hub) error { dst.Status.Size = in.Status.Size dst.Status.Namespaces = in.Status.Namespaces + dst.Status.Nodes = in.Status.Nodes switch in.Status.State { case TenantStateActive: diff --git a/api/v1beta2/tenant_status.go b/api/v1beta2/tenant_status.go index 0ef14caf..c9693ba4 100644 --- a/api/v1beta2/tenant_status.go +++ b/api/v1beta2/tenant_status.go @@ -65,6 +65,9 @@ type TenantStatusNamespaceMetadata struct { } type TenantAvailableStatus struct { + // Available Nodes within Tenant + // +optional + Nodes []string `json:"nodes,omitempty"` // Available Class Types within Tenant // +optional Classes TenantAvailableClassesStatus `json:"classes,omitzero"` diff --git a/e2e/tenant_node_status_test.go b/e2e/tenant_node_status_test.go new file mode 100644 index 00000000..d3b8b344 --- /dev/null +++ b/e2e/tenant_node_status_test.go @@ -0,0 +1,203 @@ +// Copyright 2020-2026 Project Capsule Authors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "sort" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/api" +) + +var _ = Describe("when Tenant handles Node status", Label("tenant", "nodes", "status"), func() { + const ( + e2eNodeLabelKey = "capsule.clastix.io/e2e-node-status" + e2eNodeLabelValue = "true" + ) + + tntNoRestrictions := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-node-status-no-restrictions", + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: api.OwnerListSpec{ + { + CoreOwnerSpec: api.CoreOwnerSpec{ + UserSpec: api.UserSpec{ + Name: "e2e-node-status-no-restrictions", + Kind: "User", + }, + }, + }, + }, + }, + } + + tntWithSelector := &capsulev1beta2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-node-status-with-selector", + }, + Spec: capsulev1beta2.TenantSpec{ + Owners: api.OwnerListSpec{ + { + CoreOwnerSpec: api.CoreOwnerSpec{ + UserSpec: api.UserSpec{ + Name: "e2e-node-status-with-selector", + Kind: "User", + }, + }, + }, + }, + NodeSelector: map[string]string{ + e2eNodeLabelKey: e2eNodeLabelValue, + }, + }, + } + + var ( + allNodeNames []string + primaryNode string + secondaryNode string + ) + + setNodeLabel := func(nodeName string, value *string) error { + node := &corev1.Node{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: nodeName}, node); err != nil { + return err + } + + labels := node.GetLabels() + if labels == nil { + labels = map[string]string{} + } + + if value == nil { + delete(labels, e2eNodeLabelKey) + } else { + labels[e2eNodeLabelKey] = *value + } + + node.SetLabels(labels) + + return k8sClient.Update(context.TODO(), node) + } + + JustBeforeEach(func() { + nodeList := &corev1.NodeList{} + Expect(k8sClient.List(context.TODO(), nodeList)).To(Succeed()) + Expect(nodeList.Items).ToNot(BeEmpty()) + + allNodeNames = allNodeNames[:0] + for i := range nodeList.Items { + allNodeNames = append(allNodeNames, nodeList.Items[i].GetName()) + } + sort.Strings(allNodeNames) + + primaryNode = nodeList.Items[0].GetName() + secondaryNode = "" + if len(nodeList.Items) > 1 { + secondaryNode = nodeList.Items[1].GetName() + } + + for i := range nodeList.Items { + name := nodeList.Items[i].GetName() + EventuallyCreation(func() error { + if name == primaryNode { + return setNodeLabel(name, ptrTo(e2eNodeLabelValue)) + } + + return setNodeLabel(name, nil) + }).Should(Succeed()) + } + + for _, tnt := range []*capsulev1beta2.Tenant{tntNoRestrictions, tntWithSelector} { + EventuallyCreation(func() error { + tnt.ResourceVersion = "" + return k8sClient.Create(context.TODO(), tnt) + }).Should(Succeed()) + } + }) + + JustAfterEach(func() { + for _, tnt := range []*capsulev1beta2.Tenant{tntNoRestrictions, tntWithSelector} { + EventuallyCreation(func() error { + return ignoreNotFound(k8sClient.Delete(context.TODO(), tnt)) + }).Should(Succeed()) + } + + nodeList := &corev1.NodeList{} + Expect(k8sClient.List(context.TODO(), nodeList)).To(Succeed()) + + for i := range nodeList.Items { + name := nodeList.Items[i].GetName() + EventuallyCreation(func() error { + return setNodeLabel(name, nil) + }).Should(Succeed()) + } + }) + + It("should reconcile status nodes on create and metadata update events", func() { + By("verifying initial status nodes") + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tntNoRestrictions.GetName()}, t); err != nil { + return nil, err + } + + return t.Status.Nodes, nil + }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(allNodeNames)) + + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tntWithSelector.GetName()}, t); err != nil { + return nil, err + } + + return t.Status.Nodes, nil + }, defaultTimeoutInterval, defaultPollInterval).Should(Equal([]string{primaryNode})) + + By("updating node labels to trigger metadata-based status reconciliation") + EventuallyCreation(func() error { + return setNodeLabel(primaryNode, nil) + }).Should(Succeed()) + + expectedSelectorNodes := []string{} + if secondaryNode != "" { + EventuallyCreation(func() error { + return setNodeLabel(secondaryNode, ptrTo(e2eNodeLabelValue)) + }).Should(Succeed()) + expectedSelectorNodes = append(expectedSelectorNodes, secondaryNode) + } + sort.Strings(expectedSelectorNodes) + + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tntWithSelector.GetName()}, t); err != nil { + return nil, err + } + + return t.Status.Nodes, nil + }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(expectedSelectorNodes)) + + Eventually(func() ([]string, error) { + t := &capsulev1beta2.Tenant{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: tntNoRestrictions.GetName()}, t); err != nil { + return nil, err + } + + return t.Status.Nodes, nil + }, defaultTimeoutInterval, defaultPollInterval).Should(Equal(allNodeNames)) + }) +}) + +func ptrTo(s string) *string { + return &s +} diff --git a/internal/controllers/tenant/manager.go b/internal/controllers/tenant/manager.go index 8cc70edd..c5315884 100644 --- a/internal/controllers/tenant/manager.go +++ b/internal/controllers/tenant/manager.go @@ -116,6 +116,10 @@ func (r *Manager) SetupWithManager(mgr ctrl.Manager, ctrlConfig utils.Controller ), builder.WithPredicates(predicates.UpdatedLabelsPredicate{}), ). + Watches( + &corev1.Node{}, + r.statusOnlyHandlerNodes(), + ). Watches( &capsulev1beta2.TenantOwner{}, handler.TypedFuncs[client.Object, ctrl.Request]{ diff --git a/internal/controllers/tenant/status.go b/internal/controllers/tenant/status.go index 7fbe2eec..2e252aeb 100644 --- a/internal/controllers/tenant/status.go +++ b/internal/controllers/tenant/status.go @@ -8,6 +8,7 @@ import ( "fmt" "sort" + corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" resources "k8s.io/api/resource/v1" schedulingv1 "k8s.io/api/scheduling/v1" @@ -112,9 +113,46 @@ func (r *Manager) collectAvailableResources(ctx context.Context, tnt *capsulev1b log.V(5).Info("collected available runtimeclasses", "size", len(tnt.Status.Classes.RuntimeClasses)) + if err = r.collectAvailableNodes(ctx, tnt); err != nil { + return err + } + + log.V(5).Info("collected available nodes", "size", len(tnt.Status.Nodes)) + return nil } +func (r *Manager) collectAvailableNodes(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { + nodeList := &corev1.NodeList{} + if err = r.List(ctx, nodeList); err != nil { + return err + } + + nodes := make([]string, 0, len(nodeList.Items)) + for i := range nodeList.Items { + n := &nodeList.Items[i] + if !tenantNodeSelectorMatch(tnt, n) { + continue + } + + nodes = append(nodes, n.GetName()) + } + + sort.Strings(nodes) + + tnt.Status.Nodes = nodes + + return nil +} + +func tenantNodeSelectorMatch(tnt *capsulev1beta2.Tenant, obj client.Object) bool { + if len(tnt.Spec.NodeSelector) == 0 { + return true + } + + return labels.SelectorFromSet(tnt.Spec.NodeSelector).Matches(labels.Set(obj.GetLabels())) +} + func (r *Manager) collectAvailableDeviceClasses(ctx context.Context, tnt *capsulev1beta2.Tenant) (err error) { if tnt.Status.Classes.DeviceClasses, err = listObjectNamesBySelector2( ctx, diff --git a/internal/controllers/tenant/utils.go b/internal/controllers/tenant/utils.go index ee0e2195..dd7d6b26 100644 --- a/internal/controllers/tenant/utils.go +++ b/internal/controllers/tenant/utils.go @@ -5,6 +5,7 @@ package tenant import ( "context" + "slices" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" @@ -17,6 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" + "github.com/projectcapsule/capsule/pkg/runtime/predicates" "github.com/projectcapsule/capsule/pkg/utils" ) @@ -56,6 +58,81 @@ func (r *Manager) statusOnlyHandlerClasses( } } +func (r *Manager) statusOnlyHandlerNodes() *handler.TypedFuncs[client.Object, reconcile.Request] { + metadataPredicate := predicates.UpdatedMetadataPredicate{} + + return &handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func( + ctx context.Context, + e event.TypedCreateEvent[client.Object], + _ workqueue.TypedRateLimitingInterface[reconcile.Request], + ) { + r.reconcileNodeStatusForMatchingTenants(ctx, e.Object, func(tnt *capsulev1beta2.Tenant, node client.Object) bool { + return tenantNodeSelectorMatch(tnt, node) + }) + }, + UpdateFunc: func( + ctx context.Context, + e event.TypedUpdateEvent[client.Object], + _ workqueue.TypedRateLimitingInterface[reconcile.Request], + ) { + if !metadataPredicate.Update(event.UpdateEvent{ + ObjectOld: e.ObjectOld, + ObjectNew: e.ObjectNew, + }) { + return + } + + r.reconcileNodeStatusForMatchingTenants(ctx, e.ObjectNew, func(tnt *capsulev1beta2.Tenant, node client.Object) bool { + return tenantNodeSelectorMatch(tnt, node) || slices.Contains(tnt.Status.Nodes, node.GetName()) + }) + }, + DeleteFunc: func( + ctx context.Context, + e event.TypedDeleteEvent[client.Object], + _ workqueue.TypedRateLimitingInterface[reconcile.Request], + ) { + r.reconcileNodeStatusForMatchingTenants(ctx, e.Object, func(tnt *capsulev1beta2.Tenant, node client.Object) bool { + return slices.Contains(tnt.Status.Nodes, node.GetName()) + }) + }, + } +} + +func (r *Manager) reconcileNodeStatusForMatchingTenants( + ctx context.Context, + node client.Object, + tenantMatchFn func(*capsulev1beta2.Tenant, client.Object) bool, +) { + if node == nil { + return + } + + var tenants capsulev1beta2.TenantList + if err := r.List(ctx, &tenants); err != nil { + r.Log.Error(err, "failed to list Tenants for node event") + + return + } + + for i := range tenants.Items { + tnt := &tenants.Items[i] + if !tenantMatchFn(tnt, node) { + continue + } + + if err := r.collectAvailableNodes(ctx, tnt); err != nil { + r.Log.Error(err, "cannot collect node status", "tenant", tnt.GetName(), "node", node.GetName()) + + continue + } + + if err := r.updateTenantStatus(ctx, tnt, nil); err != nil { + r.Log.Error(err, "cannot update tenant status", "tenant", tnt.GetName()) + } + } +} + func (r *Manager) enqueueTenantsForTenantOwner( ctx context.Context, tenantOwner client.Object,