From 9bf732aab38b98dd76b97bdd8a3c0cf4aa0a487d Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Thu, 15 Dec 2022 23:54:37 +0800 Subject: [PATCH] roll back code to support lower version k8s (#291) Signed-off-by: Wei Liu Signed-off-by: Wei Liu --- ...nagedcluster-registration-clusterrole.yaml | 4 +- pkg/spoke/addon/lease_controller.go | 28 +++++++++++++ pkg/spoke/addon/lease_controller_test.go | 41 +++++++++++++++++++ pkg/spoke/spokeagent.go | 1 + 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/pkg/hub/clusterrole/manifests/managedcluster-registration-clusterrole.yaml b/pkg/hub/clusterrole/manifests/managedcluster-registration-clusterrole.yaml index dc58e1c51..a799bbdec 100644 --- a/pkg/hub/clusterrole/manifests/managedcluster-registration-clusterrole.yaml +++ b/pkg/hub/clusterrole/manifests/managedcluster-registration-clusterrole.yaml @@ -6,7 +6,9 @@ rules: # Allow spoke registration agent to get/update coordination.k8s.io/lease - apiGroups: ["coordination.k8s.io"] resources: ["leases"] - resourceNames: ["managed-cluster-lease"] + # TODO for backward compatible, we do not limit the resource name + # remove this after we no longer support lower versions kubernetes (less than 1.14) + #resourceNames: ["managed-cluster-lease"] verbs: ["get", "update"] # Allow agent to get/list/watch managed cluster addons - apiGroups: ["addon.open-cluster-management.io"] diff --git a/pkg/spoke/addon/lease_controller.go b/pkg/spoke/addon/lease_controller.go index 4f203d88d..3e0167175 100644 --- a/pkg/spoke/addon/lease_controller.go +++ b/pkg/spoke/addon/lease_controller.go @@ -37,6 +37,7 @@ type managedClusterAddOnLeaseController struct { clock clock.Clock addOnClient addonclient.Interface addOnLister addonlisterv1alpha1.ManagedClusterAddOnLister + hubLeaseClient coordv1client.CoordinationV1Interface managementLeaseClient coordv1client.CoordinationV1Interface spokeLeaseClient coordv1client.CoordinationV1Interface } @@ -45,6 +46,7 @@ type managedClusterAddOnLeaseController struct { func NewManagedClusterAddOnLeaseController(clusterName string, addOnClient addonclient.Interface, addOnInformer addoninformerv1alpha1.ManagedClusterAddOnInformer, + hubLeaseClient coordv1client.CoordinationV1Interface, managementLeaseClient coordv1client.CoordinationV1Interface, spokeLeaseClient coordv1client.CoordinationV1Interface, resyncInterval time.Duration, @@ -54,6 +56,7 @@ func NewManagedClusterAddOnLeaseController(clusterName string, clock: clock.RealClock{}, addOnClient: addOnClient, addOnLister: addOnInformer.Lister(), + hubLeaseClient: hubLeaseClient, managementLeaseClient: managementLeaseClient, spokeLeaseClient: spokeLeaseClient, } @@ -126,6 +129,31 @@ func (c *managedClusterAddOnLeaseController) syncSingle(ctx context.Context, var condition metav1.Condition switch { case errors.IsNotFound(err): + // for backward compatible, for lower versions kubernetes (less than 1.14), addons update their leases on hub + // cluster, so if we cannot find addon lease on managed/management cluster, we will try to use addon hub lease. + // TODO remove this after we no longer support lower versions kubernetes (less than 1.14) + observedLease, err = c.hubLeaseClient.Leases(addOn.Namespace).Get(ctx, addOn.Name, metav1.GetOptions{}) + if err == nil { + if now.Before(observedLease.Spec.RenewTime.Add(gracePeriod)) { + // the lease is constantly updated, update its addon status to available + condition = metav1.Condition{ + Type: addonv1alpha1.ManagedClusterAddOnConditionAvailable, + Status: metav1.ConditionTrue, + Reason: "ManagedClusterAddOnLeaseUpdated", + Message: fmt.Sprintf("%s add-on is available.", addOn.Name), + } + break + } + + // the lease is not constantly updated, update its addon status to unavailable + condition = metav1.Condition{ + Type: addonv1alpha1.ManagedClusterAddOnConditionAvailable, + Status: metav1.ConditionFalse, + Reason: "ManagedClusterAddOnLeaseUpdateStopped", + Message: fmt.Sprintf("%s add-on is not available.", addOn.Name), + } + break + } condition = metav1.Condition{ Type: addonv1alpha1.ManagedClusterAddOnConditionAvailable, Status: metav1.ConditionUnknown, diff --git a/pkg/spoke/addon/lease_controller_test.go b/pkg/spoke/addon/lease_controller_test.go index 29cbb870e..dcb1af9a3 100644 --- a/pkg/spoke/addon/lease_controller_test.go +++ b/pkg/spoke/addon/lease_controller_test.go @@ -100,6 +100,7 @@ func TestSync(t *testing.T) { name string queueKey string addOns []runtime.Object + hubLeases []runtime.Object managementLeases []runtime.Object spokeLeases []runtime.Object validateActions func(t *testing.T, ctx *testinghelpers.FakeSyncContext, actions []clienttesting.Action) @@ -108,6 +109,7 @@ func TestSync(t *testing.T) { name: "bad queue key", queueKey: "test/test/test", addOns: []runtime.Object{}, + hubLeases: []runtime.Object{}, spokeLeases: []runtime.Object{}, validateActions: func(t *testing.T, ctx *testinghelpers.FakeSyncContext, actions []clienttesting.Action) { testinghelpers.AssertNoActions(t, actions) @@ -118,6 +120,7 @@ func TestSync(t *testing.T) { queueKey: "test/test", addOns: []runtime.Object{}, spokeLeases: []runtime.Object{}, + hubLeases: []runtime.Object{}, validateActions: func(t *testing.T, ctx *testinghelpers.FakeSyncContext, actions []clienttesting.Action) { testinghelpers.AssertNoActions(t, actions) }, @@ -134,6 +137,7 @@ func TestSync(t *testing.T) { InstallNamespace: "test", }, }}, + hubLeases: []runtime.Object{}, spokeLeases: []runtime.Object{}, validateActions: func(t *testing.T, ctx *testinghelpers.FakeSyncContext, actions []clienttesting.Action) { testinghelpers.AssertActions(t, actions, "get", "patch") @@ -165,6 +169,7 @@ func TestSync(t *testing.T) { InstallNamespace: "test", }, }}, + hubLeases: []runtime.Object{}, spokeLeases: []runtime.Object{ testinghelpers.NewAddOnLease("test", "test", now.Add(-5*time.Minute)), }, @@ -198,6 +203,7 @@ func TestSync(t *testing.T) { InstallNamespace: "test", }, }}, + hubLeases: []runtime.Object{}, spokeLeases: []runtime.Object{ testinghelpers.NewAddOnLease("test", "test", now), }, @@ -241,6 +247,7 @@ func TestSync(t *testing.T) { }, }, }}, + hubLeases: []runtime.Object{}, spokeLeases: []runtime.Object{ testinghelpers.NewAddOnLease("test", "test", now), }, @@ -268,6 +275,7 @@ func TestSync(t *testing.T) { }, }, }, + hubLeases: []runtime.Object{}, spokeLeases: []runtime.Object{ testinghelpers.NewAddOnLease("test1", "test1", now.Add(-5*time.Minute)), }, @@ -292,6 +300,7 @@ func TestSync(t *testing.T) { InstallNamespace: "test", }, }}, + hubLeases: []runtime.Object{}, managementLeases: []runtime.Object{ testinghelpers.NewAddOnLease("test", "test", now), }, @@ -313,6 +322,35 @@ func TestSync(t *testing.T) { } }, }, + { + name: "addon update its lease constantly (compatibility)", + queueKey: "test/test", + addOns: []runtime.Object{&addonv1alpha1.ManagedClusterAddOn{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testinghelpers.TestManagedClusterName, + Name: "test", + }, + }}, + hubLeases: []runtime.Object{testinghelpers.NewAddOnLease(testinghelpers.TestManagedClusterName, "test", now)}, + spokeLeases: []runtime.Object{}, + validateActions: func(t *testing.T, ctx *testinghelpers.FakeSyncContext, actions []clienttesting.Action) { + testinghelpers.AssertActions(t, actions, "get", "patch") + patch := actions[1].(clienttesting.PatchAction).GetPatch() + addOn := &addonv1alpha1.ManagedClusterAddOn{} + err := json.Unmarshal(patch, addOn) + if err != nil { + t.Fatal(err) + } + addOnCond := meta.FindStatusCondition(addOn.Status.Conditions, "Available") + if addOnCond == nil { + t.Errorf("expected addon available condition, but failed") + return + } + if addOnCond.Status != metav1.ConditionTrue { + t.Errorf("expected addon available condition is available, but failed") + } + }, + }, { name: "addon has customized health check", queueKey: "test/test", @@ -327,6 +365,7 @@ func TestSync(t *testing.T) { }, }, }}, + hubLeases: []runtime.Object{}, spokeLeases: []runtime.Object{}, validateActions: func(t *testing.T, ctx *testinghelpers.FakeSyncContext, actions []clienttesting.Action) { testinghelpers.AssertNoActions(t, actions) @@ -345,12 +384,14 @@ func TestSync(t *testing.T) { } } + hubClient := kubefake.NewSimpleClientset(c.hubLeases...) managementLeaseClient := kubefake.NewSimpleClientset(c.managementLeases...) spokeLeaseClient := kubefake.NewSimpleClientset(c.spokeLeases...) ctrl := &managedClusterAddOnLeaseController{ clusterName: testinghelpers.TestManagedClusterName, clock: clocktesting.NewFakeClock(time.Now()), + hubLeaseClient: hubClient.CoordinationV1(), addOnClient: addOnClient, addOnLister: addOnInformerFactory.Addon().V1alpha1().ManagedClusterAddOns().Lister(), managementLeaseClient: managementLeaseClient.CoordinationV1(), diff --git a/pkg/spoke/spokeagent.go b/pkg/spoke/spokeagent.go index b4fe924c3..f94eb51da 100644 --- a/pkg/spoke/spokeagent.go +++ b/pkg/spoke/spokeagent.go @@ -362,6 +362,7 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext o.ClusterName, addOnClient, addOnInformerFactory.Addon().V1alpha1().ManagedClusterAddOns(), + hubKubeClient.CoordinationV1(), managementKubeClient.CoordinationV1(), spokeKubeClient.CoordinationV1(), AddOnLeaseControllerSyncInterval, //TODO: this interval time should be allowed to change from outside