Cleanup orphaned resources after Cluster deletion (#540)

* adding controller reference for garbage collection, delete API lease

* added test

* fix lint

k3k-kubelet/controller/syncer/configmap.go

@@ -100,6 +100,10 @@ func (c *ConfigMapSyncer) Reconcile(ctx context.Context, req reconcile.Request)
 
 	syncedConfigMap := c.translateConfigMap(&virtualConfigMap)
 
+	if err := controllerutil.SetControllerReference(&cluster, syncedConfigMap, c.HostClient.Scheme()); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	// handle deletion
 	if !virtualConfigMap.DeletionTimestamp.IsZero() {
 		// deleting the synced configMap if exist

k3k-kubelet/controller/syncer/ingress.go

@@ -97,6 +97,7 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	}
 
 	syncedIngress := r.ingress(&virtIngress)
+
 	if err := controllerutil.SetControllerReference(&cluster, syncedIngress, r.HostClient.Scheme()); err != nil {
 		return reconcile.Result{}, err
 	}

k3k-kubelet/controller/syncer/priorityclass.go

@@ -117,6 +117,10 @@ func (r *PriorityClassSyncer) Reconcile(ctx context.Context, req reconcile.Reque
 
 	hostPriorityClass := r.translatePriorityClass(priorityClass)
 
+	if err := controllerutil.SetControllerReference(&cluster, hostPriorityClass, r.HostClient.Scheme()); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	// handle deletion
 	if !priorityClass.DeletionTimestamp.IsZero() {
 		// deleting the synced service if exists

k3k-kubelet/controller/syncer/secret.go

@@ -100,6 +100,10 @@ func (s *SecretSyncer) Reconcile(ctx context.Context, req reconcile.Request) (re
 
 	syncedSecret := s.translateSecret(&virtualSecret)
 
+	if err := controllerutil.SetControllerReference(&cluster, syncedSecret, s.HostClient.Scheme()); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	// handle deletion
 	if !virtualSecret.DeletionTimestamp.IsZero() {
 		// deleting the synced secret if exist

k3k-kubelet/controller/syncer/service.go

@@ -75,6 +75,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request
 	}
 
 	syncedService := r.service(&virtService)
+
 	if err := controllerutil.SetControllerReference(&cluster, syncedService, r.HostClient.Scheme()); err != nil {
 		return reconcile.Result{}, err
 	}

pkg/controller/cluster/cluster_finalize.go

@@ -12,7 +12,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	coordinationv1 "k8s.io/api/coordination/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 
@@ -51,6 +53,21 @@ func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster *v1beta
 		}
 	}
 
+	// delete API server lease
+	lease := &coordinationv1.Lease{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Lease",
+			APIVersion: "coordination.k8s.io/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      cluster.Name,
+			Namespace: cluster.Namespace,
+		},
+	}
+	if err := c.Client.Delete(ctx, lease); err != nil && !apierrors.IsNotFound(err) {
+		return reconcile.Result{}, err
+	}
+
 	// Remove finalizer from the cluster and update it only when all resources are cleaned up
 	if controllerutil.RemoveFinalizer(cluster, clusterFinalizerName) {
 		log.Info("Deleting Cluster removing finalizer")

pkg/controller/cluster/server/server.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
@@ -268,6 +269,10 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error)
 	if s.cluster.Spec.Persistence.Type == v1beta1.DynamicPersistenceMode {
 		persistent = true
 		pvClaim = s.setupDynamicPersistence()
+
+		if err := controllerutil.SetControllerReference(s.cluster, &pvClaim, s.client.Scheme()); err != nil {
+			return nil, err
+		}
 	}
 
 	var (

tests/cluster_sync_test.go

@@ -0,0 +1,145 @@
+package k3k_test
+
+import (
+	"context"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/rancher/k3k/k3k-kubelet/translate"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = When("a shared mode cluster is created", Ordered, Label("e2e"), func() {
+	var (
+		virtualCluster   *VirtualCluster
+		virtualConfigMap *corev1.ConfigMap
+		virtualService   *corev1.Service
+	)
+
+	BeforeAll(func() {
+		virtualCluster = NewVirtualCluster()
+
+		DeferCleanup(func() {
+			DeleteNamespaces(virtualCluster.Cluster.Namespace)
+		})
+	})
+
+	When("a ConfigMap is created in the virtual cluster", func() {
+		BeforeAll(func() {
+			ctx := context.Background()
+
+			virtualConfigMap = &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-cm",
+					Namespace: "default",
+				},
+			}
+
+			var err error
+
+			virtualConfigMap, err = virtualCluster.Client.CoreV1().ConfigMaps("default").Create(ctx, virtualConfigMap, metav1.CreateOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("is replicated in the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualConfigMap)
+
+			// check that the ConfigMap is synced in the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(err).To(Not(HaveOccurred()))
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+	})
+
+	When("a Service is created in the virtual cluster", func() {
+		BeforeAll(func() {
+			ctx := context.Background()
+
+			virtualService = &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-svc",
+					Namespace: "default",
+				},
+				Spec: corev1.ServiceSpec{
+					Type:  corev1.ServiceTypeClusterIP,
+					Ports: []corev1.ServicePort{{Port: 8888}},
+				},
+			}
+
+			var err error
+			virtualService, err = virtualCluster.Client.CoreV1().Services("default").Create(ctx, virtualService, metav1.CreateOptions{})
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("is replicated in the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualService)
+
+			// check that the ConfigMap is synced in the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().Services(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(err).To(Not(HaveOccurred()))
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+	})
+
+	When("the cluster is deleted", func() {
+		BeforeAll(func() {
+			ctx := context.Background()
+
+			By("Deleting cluster")
+
+			err := k8sClient.Delete(ctx, virtualCluster.Cluster)
+			Expect(err).To(Not(HaveOccurred()))
+		})
+
+		It("will delete the ConfigMap from the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualConfigMap)
+
+			// check that the ConfigMap is deleted from the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(apierrors.IsNotFound(err)).To(BeTrue())
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+
+		It("will delete the Service from the host cluster", func() {
+			ctx := context.Background()
+
+			hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster)
+			namespacedName := hostTranslator.NamespacedName(virtualService)
+
+			// check that the Service is deleted from the host cluster
+			Eventually(func(g Gomega) {
+				_, err := k8s.CoreV1().Services(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{})
+				g.Expect(apierrors.IsNotFound(err)).To(BeTrue())
+			}).
+				WithTimeout(time.Minute).
+				WithPolling(time.Second).
+				Should(Succeed())
+		})
+	})
+})