Isolate only virtual cluster workload Pods (#989)

* Enhance network policy to isolate synced workload pods and improve cross-cluster pod isolation handling

* Add test for label update on synced Pod to ensure isolation label persistence

* Derive host pod CIDRs dynamically for the isolation NetworkPolicy

Compute the egress-exclude CIDRs from the --cluster-cidr flag or the live
Node PodCIDR(s) via FindPodCIDRs, instead of a hardcoded guess, so cross-cluster
pod isolation is enforced against the host's real pod network. Adds unit,
integration, and e2e coverage.

* update comment

* Sort CIDR list in FindPodCIDRs function to ensure consistent order for egress rules

* Use `t.Context()` instead of `context.Background()`

Co-authored-by: Kevin McDermott <bigkevmcd@gmail.com>

* Use Ginkgo provided context

* Refactor FindPodCIDRs to use sets for CIDR collection and simplify logic

* fix lint

---------

Co-authored-by: Kevin McDermott <bigkevmcd@gmail.com>

tests/e2e/cluster_network_test.go

@@ -1,6 +1,14 @@
 package k3k_test
 
 import (
+	"context"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	networkingv1 "k8s.io/api/networking/v1"
+
+	k3kcontroller "github.com/rancher/k3k/pkg/controller"
+	"github.com/rancher/k3k/pkg/controller/policy"
 	fwk3k "github.com/rancher/k3k/tests/framework/k3k"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -92,4 +100,31 @@ var _ = When("two virtual clusters are installed", Label(e2eTestLabel), Label(ne
 		Expect(err).To(HaveOccurred())
 		Expect(stdout).To(Not(ContainSubstring("Welcome to nginx!")))
 	})
+
+	It("excludes the real host pod CIDR from the isolation NetworkPolicy, not a hardcoded guess", func(ctx context.Context) {
+		// compute the same value the controller should have derived from the live host
+		// Nodes, so this assertion doesn't rely on the host's real pod CIDR coincidentally
+		// matching a hardcoded constant
+		expectedCIDRs, err := policy.FindPodCIDRs(ctx, k8sClient, "")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(expectedCIDRs).NotTo(BeEmpty())
+
+		for _, vc := range []*VirtualCluster{cluster1, cluster2} {
+			var networkPolicy networkingv1.NetworkPolicy
+
+			key := client.ObjectKey{
+				Name:      k3kcontroller.SafeConcatNameWithPrefix(vc.Cluster.Name),
+				Namespace: vc.Cluster.Namespace,
+			}
+
+			Expect(k8sClient.Get(ctx, key, &networkPolicy)).To(Succeed())
+
+			Expect(networkPolicy.Spec.Egress[0].To).To(ContainElement(networkingv1.NetworkPolicyPeer{
+				IPBlock: &networkingv1.IPBlock{
+					CIDR:   "0.0.0.0/0",
+					Except: expectedCIDRs,
+				},
+			}))
+		}
+	})
 })

tests/e2e/cluster_pod_test.go

@@ -338,6 +338,76 @@ var _ = Context("In a shared cluster", Label(e2eTestLabel), Ordered, func() {
 		})
 	})
 
+	When("updating the labels of a synced Pod", func() {
+		var virtualPod *corev1.Pod
+
+		BeforeEach(func(ctx context.Context) {
+			p := &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "nginx-",
+					Namespace:    "default",
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{
+						Name:  "nginx",
+						Image: "nginx",
+					}},
+				},
+			}
+
+			var err error
+
+			virtualPod, err = virtualCluster.Client.CoreV1().Pods(p.Namespace).Create(ctx, p, metav1.CreateOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("should keep the clusterName isolation label on the host Pod", func(ctx context.Context) {
+			hostPodName := translator.NamespacedName(virtualPod)
+
+			By("Checking the host Pod carries the clusterName isolation label")
+
+			Eventually(func(g Gomega) {
+				hostPod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{})
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(hostPod.Labels).To(HaveKeyWithValue(translate.ClusterNameLabel, virtualCluster.Cluster.Name))
+			}).
+				WithPolling(time.Second).
+				WithTimeout(time.Minute).
+				Should(Succeed())
+
+			By("Updating a label on the virtual Pod")
+
+			var err error
+
+			virtualPod, err = virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Get(ctx, virtualPod.Name, metav1.GetOptions{})
+			Expect(err).NotTo(HaveOccurred())
+
+			if virtualPod.Labels == nil {
+				virtualPod.Labels = map[string]string{}
+			}
+
+			virtualPod.Labels["k3k.io/test"] = "updated"
+
+			virtualPod, err = virtualCluster.Client.CoreV1().Pods(virtualPod.Namespace).Update(ctx, virtualPod, metav1.UpdateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Checking the host Pod still carries the clusterName isolation label after the update")
+
+			// The label must survive the update: it is what the isolation NetworkPolicy selects on
+			// (podSelector matchExpressions: clusterName Exists). If it is dropped, the synced
+			// workload pod escapes isolation.
+			Eventually(func(g Gomega) {
+				hostPod, err := k8s.CoreV1().Pods(hostPodName.Namespace).Get(ctx, hostPodName.Name, metav1.GetOptions{})
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(hostPod.Labels).To(HaveKeyWithValue("k3k.io/test", "updated"))
+				g.Expect(hostPod.Labels).To(HaveKeyWithValue(translate.ClusterNameLabel, virtualCluster.Cluster.Name))
+			}).
+				WithPolling(time.Second).
+				WithTimeout(time.Minute).
+				Should(Succeed())
+		})
+	})
+
 	When("creating a Pod with downward API variables in environment variable", func() {
 		var virtualPod *corev1.Pod
 

tests/integration/cluster/cluster_test.go

@@ -14,6 +14,7 @@ import (
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/rancher/k3k/k3k-kubelet/translate"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	k3kcontroller "github.com/rancher/k3k/pkg/controller"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
@@ -104,6 +105,59 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
 				Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeIngress))
 
 				Expect(spec.Ingress).To(Equal([]networkingv1.NetworkPolicyIngressRule{{}}))
+
+				// the policy should only select synced workload pods, leaving the k3k infra
+				// pods (kubelet, server) unrestricted so they can reach the host API server
+				Expect(spec.PodSelector.MatchExpressions).To(ConsistOf(metav1.LabelSelectorRequirement{
+					Key:      translate.ClusterNameLabel,
+					Operator: metav1.LabelSelectorOpExists,
+				}))
+			})
+
+			When("a host Node advertises a non-default PodCIDR", func() {
+				It("excludes the real PodCIDR from the NetworkPolicy egress, not a hardcoded guess", func() {
+					node := &corev1.Node{
+						ObjectMeta: metav1.ObjectMeta{GenerateName: "node-"},
+						Spec:       corev1.NodeSpec{PodCIDR: "192.168.77.0/24"},
+					}
+
+					Expect(k8sClient.Create(ctx, node)).To(Succeed())
+					DeferCleanup(func() {
+						Expect(k8sClient.Delete(context.Background(), node)).To(Succeed())
+					})
+
+					cluster := &v1beta1.Cluster{
+						ObjectMeta: metav1.ObjectMeta{
+							GenerateName: "cluster-",
+							Namespace:    namespace,
+						},
+					}
+
+					Expect(k8sClient.Create(ctx, cluster)).To(Succeed())
+
+					expectedNetworkPolicy := &networkingv1.NetworkPolicy{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      k3kcontroller.SafeConcatNameWithPrefix(cluster.Name),
+							Namespace: cluster.Namespace,
+						},
+					}
+
+					Eventually(func(g Gomega) {
+						err := k8sClient.Get(ctx, client.ObjectKeyFromObject(expectedNetworkPolicy), expectedNetworkPolicy)
+						g.Expect(err).To(Not(HaveOccurred()))
+
+						egressPeers := expectedNetworkPolicy.Spec.Egress[0].To
+						g.Expect(egressPeers).To(ContainElement(networkingv1.NetworkPolicyPeer{
+							IPBlock: &networkingv1.IPBlock{
+								CIDR:   "0.0.0.0/0",
+								Except: []string{"192.168.77.0/24"},
+							},
+						}))
+					}).
+						WithTimeout(time.Second * 30).
+						WithPolling(time.Second).
+						Should(Succeed())
+				})
 			})
 
 			When("exposing the cluster with nodePort", func() {

tests/integration/policy/policy_test.go

@@ -14,6 +14,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/rancher/k3k/k3k-kubelet/translate"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
 	k3kcontroller "github.com/rancher/k3k/pkg/controller"
 	"github.com/rancher/k3k/pkg/controller/policy"
@@ -88,6 +89,13 @@ var _ = Describe("VirtualClusterPolicy Controller", Label("controller"), Label("
 				Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeEgress))
 				Expect(spec.PolicyTypes).To(ContainElement(networkingv1.PolicyTypeIngress))
 
+				// the policy should only select synced workload pods (which carry the
+				// ClusterNameLabel), leaving the k3k infra pods (kubelet, server) unrestricted
+				Expect(spec.PodSelector.MatchExpressions).To(ConsistOf(metav1.LabelSelectorRequirement{
+					Key:      translate.ClusterNameLabel,
+					Operator: metav1.LabelSelectorOpExists,
+				}))
+
 				// ingress should allow everything
 				Expect(spec.Ingress).To(ConsistOf(networkingv1.NetworkPolicyIngressRule{}))