Add Conditions and current status to Cluster (#408)

* Added Cluster Conditions

* added e2e tests

* fix lint

* cli polling

* update tests
This commit is contained in:
Enrico Candino
2025-07-14 15:53:37 +02:00
committed by GitHub
parent 753b31b52a
commit 2ab71df139
17 changed files with 521 additions and 105 deletions
+1 -1
View File
@@ -27,7 +27,7 @@ type PortAllocator struct {
WebhookCM *v1.ConfigMap
}
func NewPortAllocator(ctx context.Context, client ctrlruntimeclient.Client, kubeletPortRange, webhookPortRange string) (*PortAllocator, error) {
func NewPortAllocator(ctx context.Context, client ctrlruntimeclient.Client) (*PortAllocator, error) {
log := ctrl.LoggerFrom(ctx)
log.Info("starting port allocator")
+71 -36
View File
@@ -5,7 +5,7 @@ import (
"errors"
"fmt"
"net"
"reflect"
"slices"
"strings"
"time"
@@ -22,12 +22,16 @@ import (
v1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/discovery"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -54,10 +58,13 @@ const (
memberRemovalTimeout = time.Minute * 1
)
var ErrClusterValidation = errors.New("cluster validation error")
type ClusterReconciler struct {
DiscoveryClient *discovery.DiscoveryClient
Client ctrlruntimeclient.Client
Scheme *runtime.Scheme
DiscoveryClient *discovery.DiscoveryClient
Client ctrlruntimeclient.Client
Scheme *runtime.Scheme
record.EventRecorder
SharedAgentImage string
SharedAgentImagePullPolicy string
K3SImage string
@@ -66,7 +73,7 @@ type ClusterReconciler struct {
}
// Add adds a new controller to the manager
func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgentImagePullPolicy, k3SImage string, k3SImagePullPolicy string, kubeletPortRange, webhookPortRange string, maxConcurrentReconciles int) error {
func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgentImagePullPolicy, k3SImage string, k3SImagePullPolicy string, maxConcurrentReconciles int, portAllocator *agent.PortAllocator, eventRecorder record.EventRecorder) error {
discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig())
if err != nil {
return err
@@ -76,9 +83,8 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgent
return errors.New("missing shared agent image")
}
portAllocator, err := agent.NewPortAllocator(ctx, mgr.GetClient(), kubeletPortRange, webhookPortRange)
if err != nil {
return err
if eventRecorder == nil {
eventRecorder = mgr.GetEventRecorderFor(clusterController)
}
// initialize a new Reconciler
@@ -86,6 +92,7 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgent
DiscoveryClient: discoveryClient,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
EventRecorder: eventRecorder,
SharedAgentImage: sharedAgentImage,
SharedAgentImagePullPolicy: sharedAgentImagePullPolicy,
K3SImage: k3SImage,
@@ -93,10 +100,6 @@ func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage, sharedAgent
PortAllocator: portAllocator,
}
if err := mgr.Add(portAllocator.InitPortAllocatorConfig(ctx, mgr.GetClient(), kubeletPortRange, webhookPortRange)); err != nil {
return err
}
return ctrl.NewControllerManagedBy(mgr).
For(&v1alpha1.Cluster{}).
Watches(&v1.Namespace{}, namespaceEventHandler(&reconciler)).
@@ -149,27 +152,45 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
var cluster v1alpha1.Cluster
if err := c.Client.Get(ctx, req.NamespacedName, &cluster); err != nil {
return reconcile.Result{}, err
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
}
// if DeletionTimestamp is not Zero -> finalize the object
if !cluster.DeletionTimestamp.IsZero() {
return c.finalizeCluster(ctx, cluster)
return c.finalizeCluster(ctx, &cluster)
}
// add finalizers
// Set initial status if not already set
if cluster.Status.Phase == "" || cluster.Status.Phase == v1alpha1.ClusterUnknown {
cluster.Status.Phase = v1alpha1.ClusterProvisioning
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonProvisioning,
Message: "Cluster is being provisioned",
})
if err := c.Client.Status().Update(ctx, &cluster); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{Requeue: true}, nil
}
// add finalizer
if controllerutil.AddFinalizer(&cluster, clusterFinalizerName) {
if err := c.Client.Update(ctx, &cluster); err != nil {
return reconcile.Result{}, err
}
return reconcile.Result{Requeue: true}, nil
}
orig := cluster.DeepCopy()
reconcilerErr := c.reconcileCluster(ctx, &cluster)
// update Status if needed
if !reflect.DeepEqual(orig.Status, cluster.Status) {
if !equality.Semantic.DeepEqual(orig.Status, cluster.Status) {
if err := c.Client.Status().Update(ctx, &cluster); err != nil {
return reconcile.Result{}, err
}
@@ -186,7 +207,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
}
// update Cluster if needed
if !reflect.DeepEqual(orig.Spec, cluster.Spec) {
if !equality.Semantic.DeepEqual(orig.Spec, cluster.Spec) {
if err := c.Client.Update(ctx, &cluster); err != nil {
return reconcile.Result{}, err
}
@@ -196,8 +217,33 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
}
func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alpha1.Cluster) error {
err := c.reconcile(ctx, cluster)
c.updateStatus(cluster, err)
return err
}
func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1alpha1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
var ns v1.Namespace
if err := c.Client.Get(ctx, client.ObjectKey{Name: cluster.Namespace}, &ns); err != nil {
return err
}
if policyName, found := ns.Labels[policy.PolicyNameLabelKey]; found {
cluster.Status.PolicyName = policyName
var policy v1alpha1.VirtualClusterPolicy
if err := c.Client.Get(ctx, client.ObjectKey{Name: policyName}, &policy); err != nil {
return err
}
if err := c.validate(cluster, policy); err != nil {
return err
}
}
// if the Version is not specified we will try to use the same Kubernetes version of the host.
// This version is stored in the Status object, and it will not be updated if already set.
if cluster.Spec.Version == "" && cluster.Status.HostVersion == "" {
@@ -213,12 +259,6 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
cluster.Status.HostVersion = k8sVersion + "-k3s1"
}
// TODO: update status?
if err := c.validate(cluster); err != nil {
log.Error(err, "invalid change")
return nil
}
token, err := c.token(ctx, cluster)
if err != nil {
return err
@@ -257,14 +297,6 @@ func (c *ClusterReconciler) reconcileCluster(ctx context.Context, cluster *v1alp
}
}
var ns v1.Namespace
if err := c.Client.Get(ctx, client.ObjectKey{Name: cluster.Namespace}, &ns); err != nil {
return err
}
policyName := ns.Labels[policy.PolicyNameLabelKey]
cluster.Status.PolicyName = policyName
if err := c.ensureNetworkPolicy(ctx, cluster); err != nil {
return err
}
@@ -507,8 +539,8 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
log.Info("ensuring cluster service")
expectedService := server.Service(cluster)
currentService := expectedService.DeepCopy()
result, err := controllerutil.CreateOrUpdate(ctx, c.Client, currentService, func() error {
if err := controllerutil.SetControllerReference(cluster, currentService, c.Scheme); err != nil {
return err
@@ -518,7 +550,6 @@ func (c *ClusterReconciler) ensureClusterService(ctx context.Context, cluster *v
return nil
})
if err != nil {
return nil, err
}
@@ -671,9 +702,13 @@ func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1alpha1.C
return agentEnsurer.EnsureResources(ctx)
}
func (c *ClusterReconciler) validate(cluster *v1alpha1.Cluster) error {
func (c *ClusterReconciler) validate(cluster *v1alpha1.Cluster, policy v1alpha1.VirtualClusterPolicy) error {
if cluster.Name == ClusterInvalidName {
return errors.New("invalid cluster name " + cluster.Name + " no action will be taken")
return fmt.Errorf("%w: invalid cluster name %q", ErrClusterValidation, cluster.Name)
}
if cluster.Spec.Mode != policy.Spec.AllowedMode {
return fmt.Errorf("%w: mode %q is not allowed by the policy %q", ErrClusterValidation, cluster.Spec.Mode, policy.Name)
}
return nil
+17 -29
View File
@@ -10,44 +10,34 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
v1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)
func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster v1alpha1.Cluster) (reconcile.Result, error) {
func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster *v1alpha1.Cluster) (reconcile.Result, error) {
log := ctrl.LoggerFrom(ctx)
log.Info("finalizing Cluster")
// remove finalizer from the server pods and update them.
matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"})
listOpts := &ctrlruntimeclient.ListOptions{Namespace: cluster.Namespace}
matchingLabels.ApplyToList(listOpts)
// Set the Terminating phase and condition
cluster.Status.Phase = v1alpha1.ClusterTerminating
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonTerminating,
Message: "Cluster is being terminated",
})
var podList v1.PodList
if err := c.Client.List(ctx, &podList, listOpts); err != nil {
return reconcile.Result{}, ctrlruntimeclient.IgnoreNotFound(err)
}
for _, pod := range podList.Items {
if controllerutil.ContainsFinalizer(&pod, etcdPodFinalizerName) {
controllerutil.RemoveFinalizer(&pod, etcdPodFinalizerName)
if err := c.Client.Update(ctx, &pod); err != nil {
return reconcile.Result{}, err
}
}
}
if err := c.unbindClusterRoles(ctx, &cluster); err != nil {
if err := c.unbindClusterRoles(ctx, cluster); err != nil {
return reconcile.Result{}, err
}
// Dellaocate ports for kubelet and webhook if used
// Deallocate ports for kubelet and webhook if used
if cluster.Spec.Mode == v1alpha1.SharedClusterMode && cluster.Spec.MirrorHostNodes {
log.Info("dellocating ports for kubelet and webhook")
@@ -60,11 +50,9 @@ func (c *ClusterReconciler) finalizeCluster(ctx context.Context, cluster v1alpha
}
}
if controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) {
// remove finalizer from the cluster and update it.
controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName)
if err := c.Client.Update(ctx, &cluster); err != nil {
// Remove finalizer from the cluster and update it only when all resources are cleaned up
if controllerutil.RemoveFinalizer(cluster, clusterFinalizerName) {
if err := c.Client.Update(ctx, cluster); err != nil {
return reconcile.Result{}, err
}
}
+12 -12
View File
@@ -9,14 +9,13 @@ import (
"github.com/go-logr/zapr"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller/cluster"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
@@ -64,8 +63,15 @@ var _ = BeforeSuite(func() {
mgr, err := ctrl.NewManager(cfg, ctrl.Options{Scheme: scheme})
Expect(err).NotTo(HaveOccurred())
portAllocator, err := agent.NewPortAllocator(ctx, mgr.GetClient())
Expect(err).NotTo(HaveOccurred())
err = mgr.Add(portAllocator.InitPortAllocatorConfig(ctx, mgr.GetClient(), "50000-51000", "51001-52000"))
Expect(err).NotTo(HaveOccurred())
ctx, cancel = context.WithCancel(context.Background())
err = cluster.Add(ctx, mgr, "rancher/k3k-kubelet:latest", "", "rancher/k3s", "", "50000-51000", "51001-52000", 50)
err = cluster.Add(ctx, mgr, "rancher/k3k-kubelet:latest", "", "rancher/k3s", "", 50, portAllocator, &record.FakeRecorder{})
Expect(err).NotTo(HaveOccurred())
go func() {
@@ -86,13 +92,7 @@ var _ = AfterSuite(func() {
func buildScheme() *runtime.Scheme {
scheme := runtime.NewScheme()
err := corev1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = rbacv1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = appsv1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = networkingv1.AddToScheme(scheme)
err := clientgoscheme.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
err = v1alpha1.AddToScheme(scheme)
Expect(err).NotTo(HaveOccurred())
+10 -5
View File
@@ -26,9 +26,12 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
var (
namespace string
ctx context.Context
)
BeforeEach(func() {
ctx = context.Background()
createdNS := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}}
err := k8sClient.Create(context.Background(), createdNS)
Expect(err).To(Not(HaveOccurred()))
@@ -56,6 +59,8 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
Expect(cluster.Spec.Persistence.Type).To(Equal(v1alpha1.DynamicPersistenceMode))
Expect(cluster.Spec.Persistence.StorageRequestSize).To(Equal("1G"))
Expect(cluster.Status.Phase).To(Equal(v1alpha1.ClusterUnknown))
serverVersion, err := k8s.DiscoveryClient.ServerVersion()
Expect(err).To(Not(HaveOccurred()))
expectedHostVersion := fmt.Sprintf("%s-k3s1", serverVersion.GitVersion)
@@ -234,19 +239,19 @@ var _ = Describe("Cluster Controller", Label("controller"), Label("Cluster"), fu
var service v1.Service
Eventually(func() v1.ServiceType {
Eventually(func() error {
serviceKey := client.ObjectKey{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
}
err := k8sClient.Get(ctx, serviceKey, &service)
Expect(client.IgnoreNotFound(err)).To(Not(HaveOccurred()))
return service.Spec.Type
return k8sClient.Get(ctx, serviceKey, &service)
}).
WithTimeout(time.Second * 30).
WithPolling(time.Second).
Should(Equal(v1.ServiceTypeLoadBalancer))
Should(Succeed())
Expect(service.Spec.Type).To(Equal(v1.ServiceTypeLoadBalancer))
servicePorts := service.Spec.Ports
Expect(servicePorts).NotTo(BeEmpty())
+2 -5
View File
@@ -150,17 +150,14 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl
}
// remove our finalizer from the list and update it.
if controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName)
if controllerutil.RemoveFinalizer(pod, etcdPodFinalizerName) {
if err := p.Client.Update(ctx, pod); err != nil {
return err
}
}
}
if !controllerutil.ContainsFinalizer(pod, etcdPodFinalizerName) {
controllerutil.AddFinalizer(pod, etcdPodFinalizerName)
if controllerutil.AddFinalizer(pod, etcdPodFinalizerName) {
return p.Client.Update(ctx, pod)
}
+6 -1
View File
@@ -8,16 +8,21 @@ import (
"github.com/rancher/k3k/pkg/controller/cluster/agent"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
func (s *Server) Config(init bool, serviceIP string) (*v1.Secret, error) {
name := configSecretName(s.cluster.Name, init)
s.cluster.Status.TLSSANs = append(s.cluster.Spec.TLSSANs,
sans := sets.NewString(s.cluster.Spec.TLSSANs...)
sans.Insert(
serviceIP,
ServiceName(s.cluster.Name),
fmt.Sprintf("%s.%s", ServiceName(s.cluster.Name), s.cluster.Namespace),
)
s.cluster.Status.TLSSANs = sans.List()
config := serverConfigData(serviceIP, s.cluster, s.token)
if init {
config = initConfigData(s.cluster, s.token)
+7 -12
View File
@@ -49,22 +49,17 @@ func Service(cluster *v1alpha1.Cluster) *v1.Service {
if cluster.Spec.Expose != nil {
expose := cluster.Spec.Expose
// ingress
if expose.Ingress != nil {
service.Spec.Type = v1.ServiceTypeClusterIP
service.Spec.Ports = append(service.Spec.Ports, k3sServerPort, etcdPort)
}
// loadbalancer
if expose.LoadBalancer != nil {
switch {
case expose.LoadBalancer != nil:
service.Spec.Type = v1.ServiceTypeLoadBalancer
addLoadBalancerPorts(service, *expose.LoadBalancer, k3sServerPort, etcdPort)
}
// nodeport
if expose.NodePort != nil {
case expose.NodePort != nil:
service.Spec.Type = v1.ServiceTypeNodePort
addNodePortPorts(service, *expose.NodePort, k3sServerPort, etcdPort)
default:
// default to clusterIP for ingress or empty expose config
service.Spec.Type = v1.ServiceTypeClusterIP
service.Spec.Ports = append(service.Spec.Ports, k3sServerPort, etcdPort)
}
}
+95
View File
@@ -0,0 +1,95 @@
package cluster
import (
"errors"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
// Condition Types
ConditionReady = "Ready"
// Condition Reasons
ReasonValidationFailed = "ValidationFailed"
ReasonProvisioning = "Provisioning"
ReasonProvisioned = "Provisioned"
ReasonProvisioningFailed = "ProvisioningFailed"
ReasonTerminating = "Terminating"
)
func (c *ClusterReconciler) updateStatus(cluster *v1alpha1.Cluster, reconcileErr error) {
if !cluster.DeletionTimestamp.IsZero() {
cluster.Status.Phase = v1alpha1.ClusterTerminating
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonTerminating,
Message: "Cluster is being terminated",
})
return
}
// Handle validation errors specifically to set the Pending phase.
if errors.Is(reconcileErr, ErrClusterValidation) {
cluster.Status.Phase = v1alpha1.ClusterPending
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonValidationFailed,
Message: reconcileErr.Error(),
})
c.Eventf(cluster, v1.EventTypeWarning, ReasonValidationFailed, reconcileErr.Error())
return
}
if errors.Is(reconcileErr, bootstrap.ErrServerNotReady) {
cluster.Status.Phase = v1alpha1.ClusterProvisioning
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonProvisioning,
Message: reconcileErr.Error(),
})
return
}
// If there's an error, but it's not a validation error, the cluster is in a failed state.
if reconcileErr != nil {
cluster.Status.Phase = v1alpha1.ClusterFailed
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonProvisioningFailed,
Message: reconcileErr.Error(),
})
c.Eventf(cluster, v1.EventTypeWarning, ReasonProvisioningFailed, reconcileErr.Error())
return
}
// If we reach here, everything is successful.
cluster.Status.Phase = v1alpha1.ClusterReady
newCondition := metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionTrue,
Reason: ReasonProvisioned,
Message: "Cluster successfully provisioned",
}
// Only emit event on transition to Ready
if !meta.IsStatusConditionPresentAndEqual(cluster.Status.Conditions, ConditionReady, metav1.ConditionTrue) {
c.Eventf(cluster, v1.EventTypeNormal, ReasonProvisioned, newCondition.Message)
}
meta.SetStatusCondition(&cluster.Status.Conditions, newCondition)
}