github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-18 03:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for pod eviction in host cluster (#484)

Browse Source 
* update statefulset controller

* fix for single pod

* adding pod controller

* added test

* removed comment

* merged service controller

* revert statefulset

* added test

* added common owner filter
This commit is contained in:
Enrico Candino
2025-10-03 16:22:54 +02:00
committed by GitHub
parent 6523b8339b
commit 5c95ca3dfa
9 changed files with 226 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/cluster/client.go
+35
View File
@@ -0,0 +1,35 @@
package cluster
import (
"context"
"fmt"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/clientcmd"
v1 "k8s.io/api/core/v1"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/pkg/controller"
)
// newVirtualClient
func newVirtualClient(ctx context.Context, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace string) (ctrlruntimeclient.Client, error) {
var clusterKubeConfig v1.Secret
kubeconfigSecretName := types.NamespacedName{
Name: controller.SafeConcatNameWithPrefix(clusterName, "kubeconfig"),
Namespace: clusterNamespace,
}
if err := hostClient.Get(ctx, kubeconfigSecretName, &clusterKubeConfig); err != nil {
return nil, fmt.Errorf("failed to get kubeconfig secret: %w", err)
}
restConfig, err := clientcmd.RESTConfigFromKubeConfig(clusterKubeConfig.Data["kubeconfig.yaml"])
if err != nil {
return nil, fmt.Errorf("failed to create config from kubeconfig file: %w", err)
}
return ctrlruntimeclient.New(restConfig, ctrlruntimeclient.Options{})
}
pkg/controller/cluster/filter.go
+38
View File
@@ -0,0 +1,38 @@
package cluster
import (
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/predicate"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/rancher/k3k/k3k-kubelet/translate"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
)
func newClusterPredicate() predicate.Predicate {
return predicate.NewPredicateFuncs(func(object client.Object) bool {
owner := metav1.GetControllerOf(object)
return owner != nil &&
owner.Kind == "Cluster" &&
owner.APIVersion == v1alpha1.SchemeGroupVersion.String()
})
}
func clusterNamespacedName(object client.Object) types.NamespacedName {
var clusterName string
owner := metav1.GetControllerOf(object)
if owner != nil && owner.Kind == "Cluster" && owner.APIVersion == v1alpha1.SchemeGroupVersion.String() {
clusterName = owner.Name
} else {
clusterName = object.GetLabels()[translate.ClusterNameLabel]
}
return types.NamespacedName{
Name: clusterName,
Namespace: object.GetNamespace(),
}
}
pkg/controller/cluster/pod.go
+81
View File
@@ -0,0 +1,81 @@
package cluster
import (
"context"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/k3k-kubelet/translate"
)
const (
podController = "k3k-pod-controller"
)
type PodReconciler struct {
Client ctrlruntimeclient.Client
Scheme *runtime.Scheme
}
// AddPodController adds a new controller for Pods to the manager.
// It will reconcile the Pods of the Host Cluster with the one of the Virtual Cluster.
func AddPodController(ctx context.Context, mgr manager.Manager, maxConcurrentReconciles int) error {
reconciler := PodReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}
return ctrl.NewControllerManagedBy(mgr).
For(&v1.Pod{}).
Named(podController).
WithEventFilter(newClusterPredicate()).
WithOptions(controller.Options{MaxConcurrentReconciles: maxConcurrentReconciles}).
Complete(&reconciler)
}
func (r *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
log := ctrl.LoggerFrom(ctx)
log.Info("reconciling pod")
var pod v1.Pod
if err := r.Client.Get(ctx, req.NamespacedName, &pod); err != nil {
if !apierrors.IsNotFound(err) {
return reconcile.Result{}, err
}
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
}
// get cluster from the object
cluster := clusterNamespacedName(&pod)
virtualClient, err := newVirtualClient(ctx, r.Client, cluster.Name, cluster.Namespace)
if err != nil {
return reconcile.Result{}, err
}
if !pod.DeletionTimestamp.IsZero() {
virtName := pod.GetAnnotations()[translate.ResourceNameAnnotation]
virtNamespace := pod.GetAnnotations()[translate.ResourceNamespaceAnnotation]
virtPod := v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: virtName,
Namespace: virtNamespace,
},
}
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(virtualClient.Delete(ctx, &virtPod))
}
return reconcile.Result{}, nil
}
pkg/controller/cluster/service.go
+13 -51
View File
@@ -6,9 +6,7 @@ import (
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
v1 "k8s.io/api/core/v1"
@@ -16,7 +14,6 @@ import (
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/k3k-kubelet/translate"
"github.com/rancher/k3k/pkg/controller"
)
const (
@@ -35,83 +32,48 @@ func AddServiceController(ctx context.Context, mgr manager.Manager, maxConcurren
return ctrl.NewControllerManagedBy(mgr).
Named(serviceController).
For(&v1.Service{}).WithEventFilter(predicate.NewPredicateFuncs(reconciler.filterResources)).
For(&v1.Service{}).
WithEventFilter(newClusterPredicate()).
Complete(&reconciler)
}
func (r *ServiceReconciler) filterResources(object ctrlruntimeclient.Object) bool {
_, hasClusterNameLabel := object.GetLabels()[translate.ClusterNameLabel]
_, hasNameAnnotation := object.GetAnnotations()[translate.ResourceNameAnnotation]
_, hasNamespaceAnnotation := object.GetAnnotations()[translate.ResourceNamespaceAnnotation]
// Return true only if all were found
return hasClusterNameLabel && hasNameAnnotation && hasNamespaceAnnotation
}
func (r *ServiceReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
log := ctrl.LoggerFrom(ctx)
log.Info("ensuring service status to virtual cluster")
var (
virtServiceName, virtServiceNamespace string
hostService, virtService v1.Service
)
var hostService, virtService v1.Service
if err := r.HostClient.Get(ctx, req.NamespacedName, &hostService); err != nil {
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
}
// get cluster information from the object
labels := hostService.GetLabels()
// get cluster from the object
cluster := clusterNamespacedName(&hostService)
clusterName, ok := labels[translate.ClusterNameLabel]
if !ok {
return reconcile.Result{}, fmt.Errorf("cluster name label is not found")
}
clusterNamespace := hostService.GetNamespace()
virtClient, err := r.newVirtualClient(ctx, &hostService, clusterName, clusterNamespace)
virtualClient, err := newVirtualClient(ctx, r.HostClient, cluster.Name, cluster.Namespace)
if err != nil {
return reconcile.Result{}, fmt.Errorf("failed to get cluster info: %v", err)
}
virtServiceName = hostService.Annotations[translate.ResourceNameAnnotation]
virtServiceNamespace = hostService.Annotations[translate.ResourceNamespaceAnnotation]
if !hostService.DeletionTimestamp.IsZero() {
return reconcile.Result{}, nil
}
if err := virtClient.Get(ctx, types.NamespacedName{Name: virtServiceName, Namespace: virtServiceNamespace}, &virtService); err != nil {
virtualServiceKey := types.NamespacedName{
Name: hostService.Annotations[translate.ResourceNameAnnotation],
Namespace: hostService.Annotations[translate.ResourceNamespaceAnnotation],
}
if err := virtualClient.Get(ctx, virtualServiceKey, &virtService); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to get virt service: %v", err)
}
if !equality.Semantic.DeepEqual(virtService.Status.LoadBalancer, hostService.Status.LoadBalancer) {
virtService.Status.LoadBalancer = hostService.Status.LoadBalancer
if err := virtClient.Status().Update(ctx, &virtService); err != nil {
if err := virtualClient.Status().Update(ctx, &virtService); err != nil {
return reconcile.Result{}, err
}
}
return reconcile.Result{}, nil
}
func (r *ServiceReconciler) newVirtualClient(ctx context.Context, obj ctrlruntimeclient.Object, clusterName, clusterNamespace string) (ctrlruntimeclient.Client, error) {
var clusterKubeConfig v1.Secret
kubeconfigSecretName := controller.SafeConcatNameWithPrefix(clusterName, "kubeconfig")
if err := r.HostClient.Get(ctx, types.NamespacedName{Name: kubeconfigSecretName, Namespace: clusterNamespace}, &clusterKubeConfig); err != nil {
return nil, err
}
virtConfig, err := clientcmd.RESTConfigFromKubeConfig(clusterKubeConfig.Data["kubeconfig.yaml"])
if err != nil {
return nil, fmt.Errorf("failed to create config from kubeconfig file: %v", err)
}
return ctrlruntimeclient.New(virtConfig, ctrlruntimeclient.Options{})
}