✨ support singleton in hosted mode (#258)

* delete old agent when switch to singleton mode

Signed-off-by: Jian Qiu <jqiu@redhat.com>

* run singleton hosted mode

Signed-off-by: Jian Qiu <jqiu@redhat.com>

* Sync sa on spoke as external-agent-kubeconfig

Signed-off-by: Jian Qiu <jqiu@redhat.com>

---------

Signed-off-by: Jian Qiu <jqiu@redhat.com>

.github/workflows/e2e.yml

@@ -72,7 +72,7 @@ jobs:
           kind load docker-image --name=kind quay.io/open-cluster-management/addon-manager:e2e
       - name: Test E2E
         run: |
-          IMAGE_TAG=e2e KLUSTERLET_DEPLOY_MODE=Hosted make test-e2e
+          IMAGE_TAG=e2e KLUSTERLET_DEPLOY_MODE=SingletonHosted make test-e2e
         env:
           KUBECONFIG: /home/runner/.kube/config
   e2e-singleton:

manifests/klusterlet/management/klusterlet-agent-deployment.yaml

@@ -78,6 +78,10 @@ spec:
           {{if .ClusterAnnotationsString}}
           - "--cluster-annotations={{ .ClusterAnnotationsString }}"
           {{end}}
+          {{if eq .InstallMode "SingletonHosted"}}
+          - "--spoke-kubeconfig=/spoke/config/kubeconfig"
+          - "--terminate-on-files=/spoke/config/kubeconfig"
+          {{end}}
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
@@ -91,6 +95,11 @@ spec:
           readOnly: true
         - name: hub-kubeconfig
           mountPath: "/spoke/hub-kubeconfig"
+        {{if eq .InstallMode "SingletonHosted"}}
+        - name: spoke-kubeconfig-secret
+          mountPath: "/spoke/config"
+          readOnly: true
+        {{end}}
         livenessProbe:
           httpGet:
             path: /healthz
@@ -115,3 +124,8 @@ spec:
       - name: hub-kubeconfig
         emptyDir:
           medium: Memory
+      {{if eq .InstallMode "SingletonHosted"}}
+      - name: spoke-kubeconfig-secret
+        secret:
+          secretName: {{ .ExternalManagedKubeConfigAgentSecret }}
+      {{end}}

pkg/operator/helpers/helpers.go

@@ -420,7 +420,7 @@ func LoadClientConfigFromSecret(secret *corev1.Secret) (*rest.Config, error) {
 func DetermineReplica(ctx context.Context, kubeClient kubernetes.Interface, mode operatorapiv1.InstallMode, kubeVersion *version.Version) int32 {
 	// For hosted mode, there may be many cluster-manager/klusterlet running on the management cluster,
 	// set the replica to 1 to reduce the footprint of the management cluster.
-	if mode == operatorapiv1.InstallModeHosted {
+	if IsHosted(mode) {
 		return singleReplica
 	}
 
@@ -599,7 +599,7 @@ func KlusterletNamespace(klusterlet *operatorapiv1.Klusterlet) string {
 // AgentNamespace returns the namespace to deploy the agents.
 // It is on the managed cluster in the Default mode, and on the management cluster in the Hosted mode.
 func AgentNamespace(klusterlet *operatorapiv1.Klusterlet) string {
-	if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeHosted {
+	if IsHosted(klusterlet.Spec.DeployOption.Mode) {
 		return klusterlet.GetName()
 	}
 
@@ -752,3 +752,7 @@ func FeatureGateEnabled(features []operatorapiv1.FeatureGate,
 func IsSingleton(mode operatorapiv1.InstallMode) bool {
 	return mode == operatorapiv1.InstallModeSingleton || mode == operatorapiv1.InstallModeSingletonHosted
 }
+
+func IsHosted(mode operatorapiv1.InstallMode) bool {
+	return mode == operatorapiv1.InstallModeHosted || mode == operatorapiv1.InstallModeSingletonHosted
+}

pkg/operator/helpers/queuekey.go

@@ -33,6 +33,9 @@ const (
 	// ExternalManagedKubeConfigWork is the secret name of kubeconfig secret to connecting to the managed cluster
 	// Only applicable to Hosted mode, work-agent uses it to connect to the managed cluster.
 	ExternalManagedKubeConfigWork = "external-managed-kubeconfig-work"
+	// ExternalManagedKubeConfigAgent is the secret name of kubeconfig secret to connecting to the managed cluster
+	// Only applicable to SingletonHosted mode, agent uses it to connect to the managed cluster.
+	ExternalManagedKubeConfigAgent = "external-managed-kubeconfig-agent"
 
 	RegistrationWebhookSecret  = "registration-webhook-serving-cert"
 	RegistrationWebhookService = "cluster-manager-registration-webhook"
@@ -44,7 +47,7 @@ const (
 )
 
 func ClusterManagerNamespace(clustermanagername string, mode operatorapiv1.InstallMode) string {
-	if mode == operatorapiv1.InstallModeHosted {
+	if IsHosted(mode) {
 		return clustermanagername
 	}
 	return ClusterManagerDefaultNamespace

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_hub_reconcile.go

@@ -157,7 +157,7 @@ func getHubResources(mode operatorapiv1.InstallMode, config manifests.HubConfig)
 		hubResources = append(hubResources, mwReplicaSetResourceFiles...)
 	}
 	// the hubHostedWebhookServiceFiles are only used in hosted mode
-	if mode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(mode) {
 		hubResources = append(hubResources, hubHostedWebhookServiceFiles...)
 		if config.RegistrationWebhook.IsIPFormat {
 			hubResources = append(hubResources, hubHostedWebhookEndpointRegistration)

pkg/operator/operators/clustermanager/controllers/clustermanagercontroller/clustermanager_runtime_reconcile.go

@@ -76,7 +76,7 @@ func (c *runtimeReconcile) reconcile(ctx context.Context, cm *operatorapiv1.Clus
 	// In the Hosted mode, ensure the rbac kubeconfig secrets is existed for deployments to mount.
 	// In this step, we get serviceaccount token from the hub cluster to form a kubeconfig and set it as a secret on the management cluster.
 	// Before this step, the serviceaccounts in the hub cluster and the namespace in the management cluster should be applied first.
-	if cm.Spec.DeployOption.Mode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(cm.Spec.DeployOption.Mode) {
 		clusterManagerNamespace := helpers.ClusterManagerNamespace(cm.Name, cm.Spec.DeployOption.Mode)
 		err := c.ensureSAKubeconfigs(ctx, cm.Name, clusterManagerNamespace,
 			c.hubKubeConfig, c.hubKubeClient, c.kubeClient, c.recorder,

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/client_builder.go

@@ -15,6 +15,8 @@ import (
 	workclientset "open-cluster-management.io/api/client/work/clientset/versioned"
 	workv1client "open-cluster-management.io/api/client/work/clientset/versioned/typed/work/v1"
 	operatorapiv1 "open-cluster-management.io/api/operator/v1"
+
+	"open-cluster-management.io/ocm/pkg/operator/helpers"
 )
 
 type managedClusterClientsBuilderInterface interface {
@@ -70,7 +72,7 @@ func (m *managedClusterClientsBuilder) withKubeConfigSecret(namespace, name stri
 }
 
 func (m *managedClusterClientsBuilder) build(ctx context.Context) (*managedClusterClients, error) {
-	if m.mode != operatorapiv1.InstallModeHosted {
+	if !helpers.IsHosted(m.mode) {
 		return &managedClusterClients{
 			kubeClient:                m.kubeClient,
 			apiExtensionClient:        m.apiExtensionClient,

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_cleanup_controller.go

@@ -128,7 +128,7 @@ func (n *klusterletCleanupController) sync(ctx context.Context, controllerContex
 	// we should clean managedcluster resource when
 	// 1. install mode is not hosted
 	// 2. install mode is hosted and some resources has been applied on managed cluster (if hosted finalizer exists)
-	if config.InstallMode != operatorapiv1.InstallModeHosted || hasFinalizer(klusterlet, klusterletHostedFinalizer) {
+	if !helpers.IsHosted(config.InstallMode) || hasFinalizer(klusterlet, klusterletHostedFinalizer) {
 		managedClusterClients, err := n.managedClusterClientsBuilder.
 			withMode(config.InstallMode).
 			withKubeConfigSecret(config.AgentNamespace, config.ExternalManagedKubeConfigSecret).
@@ -257,7 +257,7 @@ func isTCPNoSuchHostError(err error) bool {
 // readyToAddHostedFinalizer checkes whether the hosted finalizer should be added.
 // It is only added when mode is hosted, and some resources have been applied to the managed cluster.
 func readyToAddHostedFinalizer(klusterlet *operatorapiv1.Klusterlet, mode operatorapiv1.InstallMode) bool {
-	if mode != operatorapiv1.InstallModeHosted {
+	if !helpers.IsHosted(mode) {
 		return false
 	}
 

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go

@@ -139,6 +139,7 @@ type klusterletConfig struct {
 	ExternalManagedKubeConfigSecret             string
 	ExternalManagedKubeConfigRegistrationSecret string
 	ExternalManagedKubeConfigWorkSecret         string
+	ExternalManagedKubeConfigAgentSecret        string
 	InstallMode                                 operatorapiv1.InstallMode
 
 	RegistrationFeatureGates []string
@@ -178,6 +179,7 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 		ExternalManagedKubeConfigSecret:             helpers.ExternalManagedKubeConfig,
 		ExternalManagedKubeConfigRegistrationSecret: helpers.ExternalManagedKubeConfigRegistration,
 		ExternalManagedKubeConfigWorkSecret:         helpers.ExternalManagedKubeConfigWork,
+		ExternalManagedKubeConfigAgentSecret:        helpers.ExternalManagedKubeConfigAgent,
 		InstallMode:                                 klusterlet.Spec.DeployOption.Mode,
 		HubApiServerHostAlias:                       klusterlet.Spec.HubApiServerHostAlias,
 
@@ -192,7 +194,7 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 
 	// update klusterletReadyToApply condition at first in hosted mode
 	// this conditions should be updated even when klusterlet is in deleting state.
-	if config.InstallMode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(config.InstallMode) {
 		if err != nil {
 			meta.SetStatusCondition(&klusterlet.Status.Conditions, metav1.Condition{
 				Type: klusterletReadyToApply, Status: metav1.ConditionFalse, Reason: "KlusterletPrepareFailed",

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go

@@ -420,7 +420,7 @@ func assertWorkDeployment(t *testing.T, actions []clienttesting.Action, verb, cl
 		"--agent-id=",
 	}
 
-	if mode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(mode) {
 		expectArgs = append(expectArgs,
 			"--spoke-kubeconfig=/spoke/config/kubeconfig",
 			"--terminate-on-files=/spoke/config/kubeconfig")

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_managed_reconcile.go

@@ -78,7 +78,7 @@ func (r *managedReconcile) reconcile(ctx context.Context, klusterlet *operatorap
 		return klusterlet, reconcileStop, err
 	}
 
-	if config.InstallMode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(config.InstallMode) {
 		// In hosted mode, we should ensure the namespace on the managed cluster since
 		// some resources(eg:service account) are still deployed on managed cluster.
 		err := ensureNamespace(ctx, r.managedClusterClients.kubeClient, klusterlet, config.KlusterletNamespace, r.recorder)
@@ -134,7 +134,7 @@ func (r *managedReconcile) reconcile(ctx context.Context, klusterlet *operatorap
 func (r *managedReconcile) clean(ctx context.Context, klusterlet *operatorapiv1.Klusterlet,
 	config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) {
 	// nothing should be done when deploy mode is hosted and hosted finalizer is not added.
-	if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeHosted && !hasFinalizer(klusterlet, klusterletHostedFinalizer) {
+	if helpers.IsHosted(klusterlet.Spec.DeployOption.Mode) && !hasFinalizer(klusterlet, klusterletHostedFinalizer) {
 		return klusterlet, reconcileContinue, nil
 	}
 

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_management_recocile.go

@@ -100,7 +100,7 @@ func (r *managementReconcile) clean(ctx context.Context, klusterlet *operatorapi
 	config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) {
 	// Remove secrets
 	secrets := []string{config.HubKubeConfigSecret}
-	if config.InstallMode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(config.InstallMode) {
 		// In Hosted mod, also need to remove the external-managed-kubeconfig-registration and external-managed-kubeconfig-work
 		secrets = append(secrets, []string{config.ExternalManagedKubeConfigRegistrationSecret, config.ExternalManagedKubeConfigWorkSecret}...)
 	}
@@ -121,7 +121,7 @@ func (r *managementReconcile) clean(ctx context.Context, klusterlet *operatorapi
 	// The agent namespace on the management cluster should be removed **at the end**. Otherwise if any failure occurred,
 	// the managed-external-kubeconfig secret would be removed and the next reconcile will fail due to can not build the
 	// managed cluster clients.
-	if config.InstallMode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(config.InstallMode) {
 		// remove the agent namespace on the management cluster
 		err = r.kubeClient.CoreV1().Namespaces().Delete(ctx, config.AgentNamespace, metav1.DeleteOptions{})
 		if err != nil && !errors.IsNotFound(err) {

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_runtime_reconcile.go

@@ -33,24 +33,6 @@ type runtimeReconcile struct {
 
 func (r *runtimeReconcile) reconcile(ctx context.Context, klusterlet *operatorapiv1.Klusterlet,
 	config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) {
-	if helpers.IsSingleton(config.InstallMode) {
-		return r.installSingletonAgent(ctx, klusterlet, config)
-	}
-
-	if config.InstallMode == operatorapiv1.InstallModeHosted {
-		// Create managed config secret for registration and work.
-		if err := r.createManagedClusterKubeconfig(ctx, klusterlet, config.KlusterletNamespace, config.AgentNamespace,
-			config.RegistrationServiceAccount, config.ExternalManagedKubeConfigRegistrationSecret,
-			r.recorder); err != nil {
-			return klusterlet, reconcileStop, err
-		}
-		if err := r.createManagedClusterKubeconfig(ctx, klusterlet, config.KlusterletNamespace, config.AgentNamespace,
-			config.WorkServiceAccount, config.ExternalManagedKubeConfigWorkSecret,
-			r.recorder); err != nil {
-			return klusterlet, reconcileStop, err
-		}
-	}
-
 	// Check if the klusterlet is in rebootstrapping state
 	// Both registration agent and work agent are scaled to 0 if the klusterlet is
 	// in rebootstrapping state.
@@ -59,6 +41,28 @@ func (r *runtimeReconcile) reconcile(ctx context.Context, klusterlet *operatorap
 		runtimeConfig.Replica = 0
 	}
 
+	if helpers.IsSingleton(config.InstallMode) {
+		return r.installSingletonAgent(ctx, klusterlet, runtimeConfig)
+	}
+
+	return r.installAgent(ctx, klusterlet, runtimeConfig)
+}
+
+func (r *runtimeReconcile) installAgent(ctx context.Context, klusterlet *operatorapiv1.Klusterlet,
+	runtimeConfig klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) {
+	if helpers.IsHosted(runtimeConfig.InstallMode) {
+		// Create managed config secret for registration and work.
+		if err := r.createManagedClusterKubeconfig(ctx, klusterlet, runtimeConfig.KlusterletNamespace, runtimeConfig.AgentNamespace,
+			runtimeConfig.RegistrationServiceAccount, runtimeConfig.ExternalManagedKubeConfigRegistrationSecret,
+			r.recorder); err != nil {
+			return klusterlet, reconcileStop, err
+		}
+		if err := r.createManagedClusterKubeconfig(ctx, klusterlet, runtimeConfig.KlusterletNamespace, runtimeConfig.AgentNamespace,
+			runtimeConfig.WorkServiceAccount, runtimeConfig.ExternalManagedKubeConfigWorkSecret,
+			r.recorder); err != nil {
+			return klusterlet, reconcileStop, err
+		}
+	}
 	// Deploy registration agent
 	_, generationStatus, err := helpers.ApplyDeployment(
 		ctx,
@@ -88,7 +92,7 @@ func (r *runtimeReconcile) reconcile(ctx context.Context, klusterlet *operatorap
 	// registration-agent generated the cluster name and set it into hub config secret.
 	workConfig := runtimeConfig
 	if workConfig.ClusterName == "" {
-		workConfig.ClusterName, err = r.getClusterNameFromHubKubeConfigSecret(ctx, config.AgentNamespace, klusterlet)
+		workConfig.ClusterName, err = r.getClusterNameFromHubKubeConfigSecret(ctx, runtimeConfig.AgentNamespace, klusterlet)
 		if err != nil {
 			return klusterlet, reconcileStop, err
 		}
@@ -134,9 +138,9 @@ func (r *runtimeReconcile) reconcile(ctx context.Context, klusterlet *operatorap
 	}
 
 	// clean singleton agent if there is any
-	deployments := []string{fmt.Sprintf("%s-agent", config.KlusterletName)}
+	deployments := []string{fmt.Sprintf("%s-agent", runtimeConfig.KlusterletName)}
 	for _, deployment := range deployments {
-		err := r.kubeClient.AppsV1().Deployments(config.AgentNamespace).Delete(ctx, deployment, metav1.DeleteOptions{})
+		err := r.kubeClient.AppsV1().Deployments(runtimeConfig.AgentNamespace).Delete(ctx, deployment, metav1.DeleteOptions{})
 		if err != nil && !errors.IsNotFound(err) {
 			return klusterlet, reconcileStop, err
 		}
@@ -151,13 +155,15 @@ func (r *runtimeReconcile) reconcile(ctx context.Context, klusterlet *operatorap
 
 func (r *runtimeReconcile) installSingletonAgent(ctx context.Context, klusterlet *operatorapiv1.Klusterlet,
 	config klusterletConfig) (*operatorapiv1.Klusterlet, reconcileState, error) {
-	// Check if the klusterlet is in rebootstrapping state.
-	// The agent is scaled to 0 if the klusterlet is in rebootstrapping state.
-	runtimeConfig := config
-	if meta.IsStatusConditionTrue(klusterlet.Status.Conditions, helpers.KlusterletRebootstrapProgressing) {
-		runtimeConfig.Replica = 0
+	if helpers.IsHosted(config.InstallMode) {
+		// Create managed config secret for agent. In singletonHosted mode, service account for registration/work is actually
+		// the same one, and we just pick one of them to build the external kubeconfig.
+		if err := r.createManagedClusterKubeconfig(ctx, klusterlet, config.KlusterletNamespace, config.AgentNamespace,
+			config.RegistrationServiceAccount, config.ExternalManagedKubeConfigAgentSecret,
+			r.recorder); err != nil {
+			return klusterlet, reconcileStop, err
+		}
 	}
-
 	// Deploy singleton agent
 	_, generationStatus, err := helpers.ApplyDeployment(
 		ctx,
@@ -169,7 +175,7 @@ func (r *runtimeReconcile) installSingletonAgent(ctx context.Context, klusterlet
 			if err != nil {
 				return nil, err
 			}
-			objData := assets.MustCreateAssetFromTemplate(name, template, runtimeConfig).Data
+			objData := assets.MustCreateAssetFromTemplate(name, template, config).Data
 			helpers.SetRelatedResourcesStatusesWithObj(&klusterlet.Status.RelatedResources, objData)
 			return objData, nil
 		},

test/e2e-test.mk

@@ -115,7 +115,7 @@ bootstrap-secret-hosted:
 	$(KUSTOMIZE) build deploy/klusterlet/config/samples/bootstrap | $(SED_CMD) -e "s,namespace: open-cluster-management-agent,namespace: $(KLUSTERLET_NAME)," | $(KUBECTL) apply -f -
 
 apply-spoke-cr-hosted: bootstrap-secret-hosted external-managed-secret
-	$(KUSTOMIZE) build deploy/klusterlet/config/samples | $(SED_CMD) -e "s,mode: Default,mode: Hosted," -e "s,quay.io/open-cluster-management/registration,$(REGISTRATION_IMAGE)," -e "s,quay.io/open-cluster-management/work,$(WORK_IMAGE)," -e "s,cluster1,$(MANAGED_CLUSTER_NAME)," -e "s,name: klusterlet,name: $(KLUSTERLET_NAME)," -r | $(KUBECTL) apply -f -
+	$(KUSTOMIZE) build deploy/klusterlet/config/samples | $(SED_CMD) -e "s,mode: Default,mode: SingletonHosted," -e "s,quay.io/open-cluster-management/registration,$(REGISTRATION_IMAGE)," -e "s,quay.io/open-cluster-management/work,$(WORK_IMAGE)," -e "s,quay.io/open-cluster-management/registration-operator,$(OPERATOR_IMAGE_NAME)," -e "s,cluster1,$(MANAGED_CLUSTER_NAME)," -e "s,name: klusterlet,name: $(KLUSTERLET_NAME)," -r | $(KUBECTL) apply -f -
 
 clean-hub-cr-hosted:
 	$(KUBECTL) delete managedcluster --all --ignore-not-found

test/e2e/common.go

@@ -278,7 +278,7 @@ func (t *Tester) CreateKlusterlet(name, clusterName, klusterletNamespace string,
 		}
 	}
 
-	if mode == operatorapiv1.InstallModeHosted {
+	if helpers.IsHosted(mode) {
 		// create external-managed-kubeconfig, will use the same cluster to simulate the Hosted mode.
 		secret.Namespace = agentNamespace
 		secret.Name = helpers.ExternalManagedKubeConfig

test/e2e/klusterlet_hosted_test.go

@@ -12,6 +12,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/rand"
 
 	operatorapiv1 "open-cluster-management.io/api/operator/v1"
+
+	"open-cluster-management.io/ocm/pkg/operator/helpers"
 )
 
 var _ = Describe("Delete hosted klusterlet CR", func() {
@@ -20,7 +22,7 @@ var _ = Describe("Delete hosted klusterlet CR", func() {
 	var klusterletNamespace string
 
 	BeforeEach(func() {
-		if klusterletDeployMode != string(operatorapiv1.InstallModeHosted) {
+		if !helpers.IsHosted(operatorapiv1.InstallMode(klusterletDeployMode)) {
 			Skip(fmt.Sprintf("Klusterlet deploy is %s", klusterletDeployMode))
 		}
 		klusterletName = fmt.Sprintf("e2e-klusterlet-%s", rand.String(6))