Fix Network Policy reconciliation (#388)

* logs

* fix delete cleanup

* update spec

* added policyName to status, skip netpol for policy managed clusters

pkg/controller/cluster/cluster.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rancher/k3k/pkg/controller/cluster/agent"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
+	"github.com/rancher/k3k/pkg/controller/policy"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
@@ -24,11 +25,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
-	ctrlruntimecontroller "sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
@@ -84,14 +87,47 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgent
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.Cluster{}).
-		WithOptions(ctrlruntimecontroller.Options{
-			MaxConcurrentReconciles: maxConcurrentReconciles,
-		}).
+		Watches(&v1.Namespace{}, namespaceEventHandler(&reconciler)).
 		Owns(&apps.StatefulSet{}).
 		Owns(&v1.Service{}).
 		Complete(&reconciler)
 }
 
+func namespaceEventHandler(r *ClusterReconciler) handler.Funcs {
+	return handler.Funcs{
+		// We don't need to update for create or delete events
+		CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) {},
+		DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) {},
+		// When a Namespace is updated, if it has the "policy.k3k.io/policy-name" label
+		UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) {
+			oldNs, okOld := e.ObjectOld.(*v1.Namespace)
+			newNs, okNew := e.ObjectNew.(*v1.Namespace)
+
+			if !okOld || !okNew {
+				return
+			}
+
+			oldVCPName := oldNs.Labels[policy.PolicyNameLabelKey]
+			newVCPName := newNs.Labels[policy.PolicyNameLabelKey]
+
+			// If policy hasn't changed we can skip the reconciliation
+			if oldVCPName == newVCPName {
+				return
+			}
+
+			// Enqueue all the Cluster in the namespace
+			var clusterList v1alpha1.ClusterList
+			if err := r.Client.List(ctx, &clusterList, client.InNamespace(oldNs.Name)); err != nil {
+				return
+			}
+
+			for _, cluster := range clusterList.Items {
+				q.Add(reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&cluster)})
+			}
+		},
+	}
+}
+
 func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	log := ctrl.LoggerFrom(ctx).WithValues("cluster", req.NamespacedName)
 	ctx = ctrl.LoggerInto(ctx, log) // enrich the current logger
@@ -214,6 +250,14 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
 		}
 	}
 
+	var ns v1.Namespace
+	if err := c.Client.Get(ctx, client.ObjectKey{Name: cluster.Namespace}, &ns); err != nil {
+		return err
+	}
+
+	policyName := ns.Labels[policy.PolicyNameLabelKey]
+	cluster.Status.PolicyName = policyName
+
 	if err := c.ensureNetworkPolicy(ctx, cluster); err != nil {
 		return err
 	}
@@ -320,6 +364,20 @@ func (c *ClusterReconciler) ensureNetworkPolicy(ctx context.Context, cluster *v1
 	log := ctrl.LoggerFrom(ctx)
 	log.Info("ensuring network policy")
 
+	networkPolicyName := k3kcontroller.SafeConcatNameWithPrefix(cluster.Name)
+
+	// network policies are managed by the Policy -> delete the one created as a standalone cluster
+	if cluster.Status.PolicyName != "" {
+		netpol := &networkingv1.NetworkPolicy{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      networkPolicyName,
+				Namespace: cluster.Namespace,
+			},
+		}
+
+		return ctrlruntimeclient.IgnoreNotFound(c.Client.Delete(ctx, netpol))
+	}
+
 	expectedNetworkPolicy := &networkingv1.NetworkPolicy{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      k3kcontroller.SafeConcatNameWithPrefix(cluster.Name),

pkg/controller/policy/namespace.go

@@ -50,9 +50,10 @@ func (c *VirtualClusterPolicyReconciler) cleanupNamespaces(ctx context.Context)
 	}
 
 	for _, ns := range namespaces.Items {
-		deleteOpts := []client.DeleteAllOfOption{
-			client.InNamespace(ns.Name),
-			client.MatchingLabels{ManagedByLabelKey: VirtualPolicyControllerName},
+		selector := labels.NewSelector()
+
+		if req, err := labels.NewRequirement(ManagedByLabelKey, selection.Equals, []string{VirtualPolicyControllerName}); err == nil {
+			selector = selector.Add(*req)
 		}
 
 		// if the namespace is bound to a policy -> cleanup resources of other policies
@@ -63,11 +64,15 @@ func (c *VirtualClusterPolicyReconciler) cleanupNamespaces(ctx context.Context)
 			if err != nil {
 				log.Error(err, "error creating requirement", "policy", ns.Labels[PolicyNameLabelKey])
 			} else {
-				sel := labels.NewSelector().Add(*requirement)
-				deleteOpts = append(deleteOpts, client.MatchingLabelsSelector{Selector: sel})
+				selector = selector.Add(*requirement)
 			}
 		}
 
+		deleteOpts := []client.DeleteAllOfOption{
+			client.InNamespace(ns.Name),
+			client.MatchingLabelsSelector{Selector: selector},
+		}
+
 		if err := c.Client.DeleteAllOf(ctx, &networkingv1.NetworkPolicy{}, deleteOpts...); err != nil {
 			return err
 		}

pkg/controller/policy/policy.go

@@ -66,7 +66,6 @@ func namespaceEventHandler() handler.Funcs {
 			if ns.Labels[PolicyNameLabelKey] != "" {
 				q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: ns.Labels[PolicyNameLabelKey]}})
 			}
-
 		},
 		// When a Namespace is updated, if it has the "policy.k3k.io/policy-name" label
 		UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) {