diff --git a/k3k-kubelet/controller/syncer/configmap.go b/k3k-kubelet/controller/syncer/configmap.go index e755b452..e6bfb17b 100644 --- a/k3k-kubelet/controller/syncer/configmap.go +++ b/k3k-kubelet/controller/syncer/configmap.go @@ -100,6 +100,10 @@ func (c *ConfigMapSyncer) Reconcile(ctx context.Context, req reconcile.Request) syncedConfigMap := c.translateConfigMap(&virtualConfigMap) + if err := controllerutil.SetControllerReference(&cluster, syncedConfigMap, c.HostClient.Scheme()); err != nil { + return reconcile.Result{}, err + } + // handle deletion if !virtualConfigMap.DeletionTimestamp.IsZero() { // deleting the synced configMap if exist diff --git a/k3k-kubelet/controller/syncer/ingress.go b/k3k-kubelet/controller/syncer/ingress.go index 70ae279b..94cbf033 100644 --- a/k3k-kubelet/controller/syncer/ingress.go +++ b/k3k-kubelet/controller/syncer/ingress.go @@ -97,6 +97,7 @@ func (r *IngressReconciler) Reconcile(ctx context.Context, req reconcile.Request } syncedIngress := r.ingress(&virtIngress) + if err := controllerutil.SetControllerReference(&cluster, syncedIngress, r.HostClient.Scheme()); err != nil { return reconcile.Result{}, err } diff --git a/k3k-kubelet/controller/syncer/priorityclass.go b/k3k-kubelet/controller/syncer/priorityclass.go index e4c75a8e..29bf6d9e 100644 --- a/k3k-kubelet/controller/syncer/priorityclass.go +++ b/k3k-kubelet/controller/syncer/priorityclass.go @@ -117,6 +117,10 @@ func (r *PriorityClassSyncer) Reconcile(ctx context.Context, req reconcile.Reque hostPriorityClass := r.translatePriorityClass(priorityClass) + if err := controllerutil.SetControllerReference(&cluster, hostPriorityClass, r.HostClient.Scheme()); err != nil { + return reconcile.Result{}, err + } + // handle deletion if !priorityClass.DeletionTimestamp.IsZero() { // deleting the synced service if exists diff --git a/k3k-kubelet/controller/syncer/secret.go b/k3k-kubelet/controller/syncer/secret.go index dbdc2519..835131b2 100644 --- a/k3k-kubelet/controller/syncer/secret.go +++ b/k3k-kubelet/controller/syncer/secret.go @@ -100,6 +100,10 @@ func (s *SecretSyncer) Reconcile(ctx context.Context, req reconcile.Request) (re syncedSecret := s.translateSecret(&virtualSecret) + if err := controllerutil.SetControllerReference(&cluster, syncedSecret, s.HostClient.Scheme()); err != nil { + return reconcile.Result{}, err + } + // handle deletion if !virtualSecret.DeletionTimestamp.IsZero() { // deleting the synced secret if exist diff --git a/k3k-kubelet/controller/syncer/service.go b/k3k-kubelet/controller/syncer/service.go index a4f13f19..47a2227c 100644 --- a/k3k-kubelet/controller/syncer/service.go +++ b/k3k-kubelet/controller/syncer/service.go @@ -75,6 +75,7 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request } syncedService := r.service(&virtService) + if err := controllerutil.SetControllerReference(&cluster, syncedService, r.HostClient.Scheme()); err != nil { return reconcile.Result{}, err } diff --git a/pkg/controller/cluster/cluster_finalize.go b/pkg/controller/cluster/cluster_finalize.go index 08861fd9..9f3c1ca4 100644 --- a/pkg/controller/cluster/cluster_finalize.go +++ b/pkg/controller/cluster/cluster_finalize.go @@ -12,7 +12,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" + coordinationv1 "k8s.io/api/coordination/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -51,6 +53,21 @@ func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster *v1beta } } + // delete API server lease + lease := &coordinationv1.Lease{ + TypeMeta: metav1.TypeMeta{ + Kind: "Lease", + APIVersion: "coordination.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cluster.Name, + Namespace: cluster.Namespace, + }, + } + if err := c.Client.Delete(ctx, lease); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, err + } + // Remove finalizer from the cluster and update it only when all resources are cleaned up if controllerutil.RemoveFinalizer(cluster, clusterFinalizerName) { log.Info("Deleting Cluster removing finalizer") diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 359f608a..664a082f 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -268,6 +269,10 @@ func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) if s.cluster.Spec.Persistence.Type == v1beta1.DynamicPersistenceMode { persistent = true pvClaim = s.setupDynamicPersistence() + + if err := controllerutil.SetControllerReference(s.cluster, &pvClaim, s.client.Scheme()); err != nil { + return nil, err + } } var ( diff --git a/tests/cluster_sync_test.go b/tests/cluster_sync_test.go new file mode 100644 index 00000000..44944ae5 --- /dev/null +++ b/tests/cluster_sync_test.go @@ -0,0 +1,145 @@ +package k3k_test + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/rancher/k3k/k3k-kubelet/translate" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = When("a shared mode cluster is created", Ordered, Label("e2e"), func() { + var ( + virtualCluster *VirtualCluster + virtualConfigMap *corev1.ConfigMap + virtualService *corev1.Service + ) + + BeforeAll(func() { + virtualCluster = NewVirtualCluster() + + DeferCleanup(func() { + DeleteNamespaces(virtualCluster.Cluster.Namespace) + }) + }) + + When("a ConfigMap is created in the virtual cluster", func() { + BeforeAll(func() { + ctx := context.Background() + + virtualConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cm", + Namespace: "default", + }, + } + + var err error + + virtualConfigMap, err = virtualCluster.Client.CoreV1().ConfigMaps("default").Create(ctx, virtualConfigMap, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + }) + + It("is replicated in the host cluster", func() { + ctx := context.Background() + + hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster) + namespacedName := hostTranslator.NamespacedName(virtualConfigMap) + + // check that the ConfigMap is synced in the host cluster + Eventually(func(g Gomega) { + _, err := k8s.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) + }) + + When("a Service is created in the virtual cluster", func() { + BeforeAll(func() { + ctx := context.Background() + + virtualService = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-svc", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{{Port: 8888}}, + }, + } + + var err error + virtualService, err = virtualCluster.Client.CoreV1().Services("default").Create(ctx, virtualService, metav1.CreateOptions{}) + Expect(err).To(Not(HaveOccurred())) + }) + + It("is replicated in the host cluster", func() { + ctx := context.Background() + + hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster) + namespacedName := hostTranslator.NamespacedName(virtualService) + + // check that the ConfigMap is synced in the host cluster + Eventually(func(g Gomega) { + _, err := k8s.CoreV1().Services(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{}) + g.Expect(err).To(Not(HaveOccurred())) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) + }) + + When("the cluster is deleted", func() { + BeforeAll(func() { + ctx := context.Background() + + By("Deleting cluster") + + err := k8sClient.Delete(ctx, virtualCluster.Cluster) + Expect(err).To(Not(HaveOccurred())) + }) + + It("will delete the ConfigMap from the host cluster", func() { + ctx := context.Background() + + hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster) + namespacedName := hostTranslator.NamespacedName(virtualConfigMap) + + // check that the ConfigMap is deleted from the host cluster + Eventually(func(g Gomega) { + _, err := k8s.CoreV1().ConfigMaps(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{}) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) + + It("will delete the Service from the host cluster", func() { + ctx := context.Background() + + hostTranslator := translate.NewHostTranslator(virtualCluster.Cluster) + namespacedName := hostTranslator.NamespacedName(virtualService) + + // check that the Service is deleted from the host cluster + Eventually(func(g Gomega) { + _, err := k8s.CoreV1().Services(namespacedName.Namespace).Get(ctx, namespacedName.Name, metav1.GetOptions{}) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(Succeed()) + }) + }) +})