Add hosted mode in klusterlet (#220)

Signed-off-by: xuezhaojun <zxue@redhat.com>
This commit is contained in:
xuezhaojun
2022-03-15 17:03:53 +08:00
committed by GitHub
parent 84e12c9197
commit 7aa41cda65
13 changed files with 202 additions and 66 deletions

View File

@@ -47,7 +47,7 @@ kustomize_dir:=$(dir $(KUSTOMIZE))
KUBECTL?=kubectl
KUBECONFIG?=./.kubeconfig
HUB_KUBECONFIG?=./.hub-kubeconfig
DETACHED_CLUSTER_MANAGER_NAME?=cluster-manager
HOSTED_CLUSTER_MANAGER_NAME?=cluster-manager
EXTERNAL_HUB_KUBECONFIG?=./.external-hub-kubeconfig
EXTERNAL_MANAGED_KUBECONFIG?=./.external-managed-kubeconfig
MANAGED_CLUSTER_NAME ?= cluster1
@@ -138,8 +138,8 @@ bootstrap-secret-hosted:
external-hub-secret:
cp $(EXTERNAL_HUB_KUBECONFIG) deploy/cluster-manager/config/samples/cluster-manager/external-hub-kubeconfig
$(KUBECTL) get ns $(DETACHED_CLUSTER_MANAGER_NAME); if [ $$? -ne 0 ] ; then $(KUBECTL) create ns $(DETACHED_CLUSTER_MANAGER_NAME); fi
$(KUSTOMIZE) build deploy/cluster-manager/config/samples/cluster-manager | $(SED_CMD) -e "s,cluster-manager,$(DETACHED_CLUSTER_MANAGER_NAME)," | $(KUBECTL) apply -f -
$(KUBECTL) get ns $(HOSTED_CLUSTER_MANAGER_NAME); if [ $$? -ne 0 ] ; then $(KUBECTL) create ns $(HOSTED_CLUSTER_MANAGER_NAME); fi
$(KUSTOMIZE) build deploy/cluster-manager/config/samples/cluster-manager | $(SED_CMD) -e "s,cluster-manager,$(HOSTED_CLUSTER_MANAGER_NAME)," | $(KUBECTL) apply -f -
external-managed-secret:
cp $(EXTERNAL_MANAGED_KUBECONFIG) deploy/klusterlet/config/samples/managedcluster/external-managed-kubeconfig
@@ -166,8 +166,8 @@ clean-hub-cr:
clean-hub-cr-hosted:
$(KUBECTL) delete managedcluster --all --ignore-not-found
$(KUSTOMIZE) build deploy/cluster-manager/config/samples | $(SED_CMD) -e "s,cluster-manager,$(DETACHED_CLUSTER_MANAGER_NAME)," | $(KUBECTL) delete --ignore-not-found -f -
$(KUSTOMIZE) build deploy/cluster-manager/config/samples/cluster-manager | $(SED_CMD) -e "s,cluster-manager,$(DETACHED_CLUSTER_MANAGER_NAME)," | $(KUBECTL) delete --ignore-not-found -f -
$(KUSTOMIZE) build deploy/cluster-manager/config/samples | $(SED_CMD) -e "s,cluster-manager,$(HOSTED_CLUSTER_MANAGER_NAME)," | $(KUBECTL) delete --ignore-not-found -f -
$(KUSTOMIZE) build deploy/cluster-manager/config/samples/cluster-manager | $(SED_CMD) -e "s,cluster-manager,$(HOSTED_CLUSTER_MANAGER_NAME)," | $(KUBECTL) delete --ignore-not-found -f -
clean-hub-operator:
$(KUSTOMIZE) build deploy/cluster-manager/config | $(KUBECTL) delete --ignore-not-found -f -

View File

@@ -121,7 +121,7 @@ We mainly provide deployment in two scenarios:
4. Switch to management cluster and deploy hub components.
```shell
kubectl config use-context {management-context}
make deploy-hub-detached
make deploy-hub-hosted
```
After deploy hub successfully, the user needs to expose webhook-servers in the management cluster manually.
@@ -181,7 +181,7 @@ We support deploy the Klusterlet(registration-agent, work-agent) outside of mana
3. Switch to management context and deploy agent components on management cluster.
```
kubectl config use-context {management-context}
make deploy-spoke-detached
make deploy-spoke-hosted
```
**PLEASE NOTE**: If you're running kubernetes in docker, the `server` address in kubeconfig may not be accessible for other clusters. In this case, you need to set `EXTERNAL_MANAGED_KUBECONFIG` explicitly.
@@ -199,7 +199,7 @@ We support deploy the Klusterlet(registration-agent, work-agent) outside of mana
5. To clean the spoke environment.
```shell
kubectl config use-context {management-context}
make clean-spoke-detached
make clean-spoke-hosted
## What is next

View File

@@ -51,7 +51,7 @@ spec:
{{if .ExternalServerURL}}
- "--spoke-external-server-urls={{ .ExternalServerURL }}"
{{end}}
{{if eq .InstallMode "Detached"}}
{{if eq .InstallMode "Hosted"}}
- "--spoke-kubeconfig=/spoke/config/kubeconfig"
{{end}}
{{if eq .Replica 1}}
@@ -70,7 +70,7 @@ spec:
readOnly: true
- name: hub-kubeconfig
mountPath: "/spoke/hub-kubeconfig"
{{if eq .InstallMode "Detached"}}
{{if eq .InstallMode "Hosted"}}
- name: spoke-kubeconfig-secret
mountPath: "/spoke/config"
readOnly: true
@@ -99,7 +99,7 @@ spec:
- name: hub-kubeconfig
emptyDir:
medium: Memory
{{if eq .InstallMode "Detached"}}
{{if eq .InstallMode "Hosted"}}
- name: spoke-kubeconfig-secret
secret:
secretName: {{ .ExternalManagedKubeConfigRegistrationSecret }}

View File

@@ -47,7 +47,7 @@ spec:
- "agent"
- "--spoke-cluster-name={{ .ClusterName }}"
- "--hub-kubeconfig=/spoke/hub-kubeconfig/kubeconfig"
{{if eq .InstallMode "Detached"}}
{{if eq .InstallMode "Hosted"}}
- "--spoke-kubeconfig=/spoke/config/kubeconfig"
{{end}}
{{if eq .Replica 1}}
@@ -64,7 +64,7 @@ spec:
- name: hub-kubeconfig-secret
mountPath: "/spoke/hub-kubeconfig"
readOnly: true
{{if eq .InstallMode "Detached"}}
{{if eq .InstallMode "Hosted"}}
- name: spoke-kubeconfig-secret
mountPath: "/spoke/config"
readOnly: true
@@ -90,7 +90,7 @@ spec:
- name: hub-kubeconfig-secret
secret:
secretName: {{ .HubKubeConfigSecret }}
{{if eq .InstallMode "Detached"}}
{{if eq .InstallMode "Hosted"}}
- name: spoke-kubeconfig-secret
secret:
secretName: {{ .ExternalManagedKubeConfigWorkSecret }}

View File

@@ -600,7 +600,7 @@ func LoadClientConfigFromSecret(secret *corev1.Secret) (*rest.Config, error) {
func DetermineReplica(ctx context.Context, kubeClient kubernetes.Interface, mode operatorapiv1.InstallMode) int32 {
// For hosted mode, there may be many cluster-manager/klusterlet running on the management cluster,
// set the replica to 1 to reduce the footprint of the management cluster.
if mode == operatorapiv1.InstallModeDetached {
if mode == operatorapiv1.InstallModeDetached || mode == operatorapiv1.InstallModeHosted {
return singleReplica
}
return DetermineReplicaByNodes(ctx, kubeClient)
@@ -736,9 +736,9 @@ func UpdateKlusterletRelatedResourcesFn(relatedResources ...operatorapiv1.Relate
}
// KlusterletNamespace returns the klusterletNamespace to deploy the agents.
// Note in Detached mode, the specNamespace will be ignored.
// Note in Hosted mode, the specNamespace will be ignored.
func KlusterletNamespace(klusterlet *operatorapiv1.Klusterlet) string {
if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeDetached {
if klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeDetached || klusterlet.Spec.DeployOption.Mode == operatorapiv1.InstallModeHosted {
return klusterlet.GetName()
}

View File

@@ -804,13 +804,13 @@ func TestDeterminReplica(t *testing.T) {
},
{
name: "single node hosted mode",
mode: operatorapiv1.InstallModeDetached,
mode: operatorapiv1.InstallModeHosted,
existingNodes: []runtime.Object{newNode("node1")},
expectedReplica: singleReplica,
},
{
name: "multiple node hosted mode",
mode: operatorapiv1.InstallModeDetached,
mode: operatorapiv1.InstallModeHosted,
existingNodes: []runtime.Object{newNode("node1"), newNode("node2"), newNode("node3")},
expectedReplica: singleReplica,
},
@@ -1292,27 +1292,27 @@ func TestKlusterletNamespace(t *testing.T) {
expect: "open-cluster-management-test",
},
{
name: "Detached mode with spec namespace",
name: "Hosted mode with spec namespace",
klusterlet: &operatorapiv1.Klusterlet{
ObjectMeta: metav1.ObjectMeta{
Name: "klusterlet",
},
Spec: operatorapiv1.KlusterletSpec{
Namespace: "open-cluster-management-test",
DeployOption: operatorapiv1.KlusterletDeployOption{Mode: operatorapiv1.InstallModeDetached},
DeployOption: operatorapiv1.KlusterletDeployOption{Mode: operatorapiv1.InstallModeHosted},
},
},
expect: "klusterlet",
},
{
name: "Detached mode without spec namespace",
name: "Hosted mode without spec namespace",
klusterlet: &operatorapiv1.Klusterlet{
ObjectMeta: metav1.ObjectMeta{
Name: "klusterlet",
},
Spec: operatorapiv1.KlusterletSpec{
Namespace: "",
DeployOption: operatorapiv1.KlusterletDeployOption{Mode: operatorapiv1.InstallModeDetached},
DeployOption: operatorapiv1.KlusterletDeployOption{Mode: operatorapiv1.InstallModeHosted},
},
},
expect: "klusterlet",

View File

@@ -99,8 +99,8 @@ type klusterletController struct {
skipHubSecretPlaceholder bool
cache resourceapply.ResourceCache
// buildManagedClusterClientsDetachedMode build clients for manged cluster in detached mode, this can be override for testing
buildManagedClusterClientsDetachedMode func(ctx context.Context, kubeClient kubernetes.Interface, namespace, secret string) (*managedClusterClients, error)
// buildManagedClusterClientsHostedMode build clients for manged cluster in hosted mode, this can be override for testing
buildManagedClusterClientsHostedMode func(ctx context.Context, kubeClient kubernetes.Interface, namespace, secret string) (*managedClusterClients, error)
}
// NewKlusterletController construct klusterlet controller
@@ -118,17 +118,17 @@ func NewKlusterletController(
recorder events.Recorder,
skipHubSecretPlaceholder bool) factory.Controller {
controller := &klusterletController{
kubeClient: kubeClient,
apiExtensionClient: apiExtensionClient,
dynamicClient: dynamicClient,
klusterletClient: klusterletClient,
klusterletLister: klusterletInformer.Lister(),
appliedManifestWorkClient: appliedManifestWorkClient,
kubeVersion: kubeVersion,
operatorNamespace: operatorNamespace,
buildManagedClusterClientsDetachedMode: buildManagedClusterClientsFromSecret,
skipHubSecretPlaceholder: skipHubSecretPlaceholder,
cache: resourceapply.NewResourceCache(),
kubeClient: kubeClient,
apiExtensionClient: apiExtensionClient,
dynamicClient: dynamicClient,
klusterletClient: klusterletClient,
klusterletLister: klusterletInformer.Lister(),
appliedManifestWorkClient: appliedManifestWorkClient,
kubeVersion: kubeVersion,
operatorNamespace: operatorNamespace,
buildManagedClusterClientsHostedMode: buildManagedClusterClientsFromSecret,
skipHubSecretPlaceholder: skipHubSecretPlaceholder,
cache: resourceapply.NewResourceCache(),
}
return factory.New().WithSync(controller.sync).
@@ -166,7 +166,7 @@ type managedClusterClients struct {
apiExtensionClient apiextensionsclient.Interface
appliedManifestWorkClient workv1client.AppliedManifestWorkInterface
dynamicClient dynamic.Interface
// Only used for Detached mode to generate managed cluster kubeconfig
// Only used for Hosted mode to generate managed cluster kubeconfig
// with minimum permission for registration and work.
kubeconfig *rest.Config
}
@@ -209,8 +209,13 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
appliedManifestWorkClient: n.appliedManifestWorkClient,
}
// TODO: remove this when detached mode is not used in klusterlet
if config.InstallMode == operatorapiv1.InstallModeDetached {
managedClusterClients, err = n.buildManagedClusterClientsDetachedMode(ctx, n.kubeClient, config.KlusterletNamespace, config.ExternalManagedKubeConfigSecret)
config.InstallMode = operatorapiv1.InstallModeHosted
}
if config.InstallMode == operatorapiv1.InstallModeHosted {
managedClusterClients, err = n.buildManagedClusterClientsHostedMode(ctx, n.kubeClient, config.KlusterletNamespace, config.ExternalManagedKubeConfigSecret)
if err != nil {
_, _, _ = helpers.UpdateKlusterletStatus(ctx, n.klusterletClient, klusterletName, helpers.UpdateKlusterletConditionFn(metav1.Condition{
Type: klusterletReadyToApply, Status: metav1.ConditionFalse, Reason: "KlusterletPrepareFailed",
@@ -261,9 +266,9 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
if err != nil {
return err
}
// For now, whether in Default or Detached mode, the addons will be deployed on the managed cluster.
// For now, whether in Default or Hosted mode, the addons will be deployed on the managed cluster.
// sync image pull secret from management cluster to managed cluster for addon namespace
// TODO(zhujian7): In the future, we may consider deploy addons on the management cluster in Detached mode.
// TODO(zhujian7): In the future, we may consider deploy addons on the management cluster in Hosted mode.
addonNamespace := fmt.Sprintf("%s-addon", config.KlusterletNamespace)
// Ensure the klusterlet addon namespace
err = n.ensureNamespace(ctx, managedClusterClients.kubeClient, klusterletName, addonNamespace)
@@ -276,8 +281,8 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
return err
}
if config.InstallMode == operatorapiv1.InstallModeDetached {
// In detached mode, we should ensure the namespace on the managed cluster since
if config.InstallMode == operatorapiv1.InstallModeHosted {
// In hosted mode, we should ensure the namespace on the managed cluster since
// some resources(eg:service account) are still deployed on managed cluster.
err := n.ensureNamespace(ctx, managedClusterClients.kubeClient, klusterletName, config.KlusterletNamespace)
if err != nil {
@@ -321,7 +326,7 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
}
relatedResources = append(relatedResources, statuses...)
if config.InstallMode == operatorapiv1.InstallModeDetached {
if config.InstallMode == operatorapiv1.InstallModeHosted {
// Create managed config secret for registration and work.
err = n.createManagedClusterKubeconfig(ctx, klusterletName, config.KlusterletNamespace, registrationServiceAccountName(klusterletName), config.ExternalManagedKubeConfigRegistrationSecret,
managedClusterClients.kubeconfig, managedClusterClients.kubeClient, n.kubeClient.CoreV1(), controllerContext.Recorder())
@@ -619,8 +624,8 @@ func (n *klusterletController) cleanUp(
// Remove secrets
secrets := []string{config.HubKubeConfigSecret}
if config.InstallMode == operatorapiv1.InstallModeDetached {
// In Detached mod, also need to remove the external-managed-kubeconfig-registration and external-managed-kubeconfig-work
if config.InstallMode == operatorapiv1.InstallModeHosted {
// In Hosted mod, also need to remove the external-managed-kubeconfig-registration and external-managed-kubeconfig-work
secrets = append(secrets, []string{config.ExternalManagedKubeConfigRegistrationSecret, config.ExternalManagedKubeConfigWorkSecret}...)
}
for _, secret := range secrets {
@@ -656,8 +661,8 @@ func (n *klusterletController) cleanUp(
}
// remove the klusterlet namespace and klusterlet addon namespace on the managed cluster
// For now, whether in Default or Detached mode, the addons will be deployed on the managed cluster.
// TODO(zhujian7): In the future, we may consider deploy addons on the management cluster in Detached mode.
// For now, whether in Default or Hosted mode, the addons will be deployed on the managed cluster.
// TODO(zhujian7): In the future, we may consider deploy addons on the management cluster in Hosted mode.
namespaces := []string{config.KlusterletNamespace, fmt.Sprintf("%s-addon", config.KlusterletNamespace)}
for _, namespace := range namespaces {
err = managedClients.kubeClient.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{})
@@ -683,7 +688,7 @@ func (n *klusterletController) cleanUp(
// The klusterlet namespace on the management cluster should be removed **at the end**. Otherwise if any failure occurred,
// the managed-external-kubeconfig secret would be removed and the next reconcile will fail due to can not build the
// managed cluster clients.
if config.InstallMode == operatorapiv1.InstallModeDetached {
if config.InstallMode == operatorapiv1.InstallModeHosted {
// remove the klusterlet namespace on the management cluster
err = n.kubeClient.CoreV1().Namespaces().Delete(ctx, config.KlusterletNamespace, metav1.DeleteOptions{})
if err != nil && !errors.IsNotFound(err) {
@@ -809,7 +814,7 @@ func getServersFromKlusterlet(klusterlet *operatorapiv1.Klusterlet) string {
return strings.Join(serverString, ",")
}
// getManagedKubeConfig is a helper func for Detached mode, it will retrive managed cluster
// getManagedKubeConfig is a helper func for Hosted mode, it will retrive managed cluster
// kubeconfig from "external-managed-kubeconfig" secret.
func getManagedKubeConfig(ctx context.Context, kubeClient kubernetes.Interface, namespace, secretName string) (*rest.Config, error) {
managedKubeconfigSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})

View File

@@ -85,9 +85,9 @@ func newKlusterlet(name, namespace, clustername string) *opratorapiv1.Klusterlet
}
}
func newKlusterletDetached(name, namespace, clustername string) *opratorapiv1.Klusterlet {
func newKlusterletHosted(name, namespace, clustername string) *opratorapiv1.Klusterlet {
klusterlet := newKlusterlet(name, namespace, clustername)
klusterlet.Spec.DeployOption.Mode = opratorapiv1.InstallModeDetached
klusterlet.Spec.DeployOption.Mode = opratorapiv1.InstallModeHosted
return klusterlet
}
@@ -162,7 +162,7 @@ func newTestController(klusterlet *opratorapiv1.Klusterlet, appliedManifestWorks
}
}
func newTestControllerDetached(klusterlet *opratorapiv1.Klusterlet, appliedManifestWorks []runtime.Object, objects ...runtime.Object) *testController {
func newTestControllerHosted(klusterlet *opratorapiv1.Klusterlet, appliedManifestWorks []runtime.Object, objects ...runtime.Object) *testController {
fakeKubeClient := fakekube.NewSimpleClientset(objects...)
fakeAPIExtensionClient := fakeapiextensions.NewSimpleClientset()
fakeOperatorClient := fakeoperatorclient.NewSimpleClientset(klusterlet)
@@ -225,7 +225,7 @@ func newTestControllerDetached(klusterlet *opratorapiv1.Klusterlet, appliedManif
kubeVersion: kubeVersion,
operatorNamespace: "open-cluster-management",
cache: resourceapply.NewResourceCache(),
buildManagedClusterClientsDetachedMode: func(ctx context.Context, kubeClient kubernetes.Interface, namespace, secret string) (*managedClusterClients, error) {
buildManagedClusterClientsHostedMode: func(ctx context.Context, kubeClient kubernetes.Interface, namespace, secret string) (*managedClusterClients, error) {
return &managedClusterClients{
kubeClient: fakeManagedKubeClient,
apiExtensionClient: fakeManagedAPIExtensionClient,
@@ -336,7 +336,7 @@ func assertWorkDeployment(t *testing.T, actions []clienttesting.Action, verb, cl
"--hub-kubeconfig=/spoke/hub-kubeconfig/kubeconfig",
}
if mode == opratorapiv1.InstallModeDetached {
if mode == opratorapiv1.InstallModeDetached || mode == opratorapiv1.InstallModeHosted {
expectArgs = append(expectArgs, "--spoke-kubeconfig=/spoke/config/kubeconfig")
}
@@ -439,10 +439,10 @@ func TestSyncDeploy(t *testing.T) {
testinghelper.NamedCondition(klusterletApplied, "KlusterletApplied", metav1.ConditionTrue))
}
// TestSyncDeployDetached test deployment of klusterlet components in detached mode
func TestSyncDeployDetached(t *testing.T) {
// TestSyncDeployHosted test deployment of klusterlet components in hosted mode
func TestSyncDeployHosted(t *testing.T) {
installedNamespace := "klusterlet"
klusterlet := newKlusterletDetached("klusterlet", "testns", "cluster1")
klusterlet := newKlusterletHosted("klusterlet", "testns", "cluster1")
bootStrapSecret := newSecret(helpers.BootstrapHubKubeConfig, installedNamespace)
hubKubeConfigSecret := newSecret(helpers.HubKubeConfig, installedNamespace)
hubKubeConfigSecret.Data["kubeconfig"] = []byte("dummuykubeconnfig")
@@ -450,7 +450,7 @@ func TestSyncDeployDetached(t *testing.T) {
// externalManagedSecret.Data["kubeconfig"] = []byte("dummuykubeconnfig")
namespace := newNamespace(installedNamespace)
pullSecret := newSecret(imagePullSecret, "open-cluster-management")
controller := newTestControllerDetached(klusterlet, nil, bootStrapSecret, hubKubeConfigSecret, namespace, pullSecret /*externalManagedSecret*/)
controller := newTestControllerHosted(klusterlet, nil, bootStrapSecret, hubKubeConfigSecret, namespace, pullSecret /*externalManagedSecret*/)
syncContext := testinghelper.NewFakeSyncContext(t, "klusterlet")
err := controller.controller.sync(context.TODO(), syncContext)
@@ -604,8 +604,8 @@ func TestSyncDelete(t *testing.T) {
}
}
func TestSyncDeleteDetached(t *testing.T) {
klusterlet := newKlusterletDetached("klusterlet", "testns", "cluster1")
func TestSyncDeleteHosted(t *testing.T) {
klusterlet := newKlusterletHosted("klusterlet", "testns", "cluster1")
now := metav1.Now()
klusterlet.ObjectMeta.SetDeletionTimestamp(&now)
installedNamespace := helpers.KlusterletNamespace(klusterlet)
@@ -619,7 +619,7 @@ func TestSyncDeleteDetached(t *testing.T) {
newAppliedManifestWorks("testhost", []string{appliedManifestWorkFinalizer}, true),
newAppliedManifestWorks("testhost-2", []string{appliedManifestWorkFinalizer}, false),
}
controller := newTestControllerDetached(klusterlet, appliedManifestWorks, bootstrapKubeConfigSecret, namespace /*externalManagedSecret*/)
controller := newTestControllerHosted(klusterlet, appliedManifestWorks, bootstrapKubeConfigSecret, namespace /*externalManagedSecret*/)
syncContext := testinghelper.NewFakeSyncContext(t, klusterlet.Name)
err := controller.controller.sync(context.TODO(), syncContext)

View File

@@ -65,6 +65,7 @@ var _ = Describe("Manage the managed cluster addons", func() {
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
})
// TODO: remove this after the detached mode is not used in klusterlet
It("Create one managed cluster addon and make sure it is available in Detached mode", func() {
var err error
By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName))
@@ -105,4 +106,45 @@ var _ = Describe("Manage the managed cluster addons", func() {
return t.CheckManagedClusterAddOnStatus(clusterName, addOnName)
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
})
It("Create one managed cluster addon and make sure it is available in Hosted mode", func() {
var err error
By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName))
_, err = t.CreateKlusterlet(klusterletName, clusterName, agentNamespace, operatorapiv1.InstallModeHosted)
Expect(err).ToNot(HaveOccurred())
By(fmt.Sprintf("waiting for the managed cluster %v to be created", clusterName))
Eventually(func() error {
_, err = t.GetCreatedManagedCluster(clusterName)
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("approve the created managed cluster %v", clusterName))
Eventually(func() error {
return t.ApproveCSR(clusterName)
}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
By(fmt.Sprintf("accept the created managed cluster %v", clusterName))
Eventually(func() error {
return t.AcceptsClient(clusterName)
}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
By(fmt.Sprintf("waiting for the managed cluster %v to be ready", clusterName))
Eventually(func() error {
return t.CheckManagedClusterStatus(clusterName)
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("create the addon %v on the managed cluster namespace %v", addOnName, clusterName))
err = t.CreateManagedClusterAddOn(clusterName, addOnName)
Expect(err).ToNot(HaveOccurred())
By(fmt.Sprintf("create the addon lease %v on addon install namespace %v", addOnName, addOnName))
err = t.CreateManagedClusterAddOnLease(addOnName, addOnName)
Expect(err).ToNot(HaveOccurred())
By(fmt.Sprintf("wait the addon %v available condition to be true", addOnName))
Eventually(func() error {
return t.CheckManagedClusterAddOnStatus(clusterName, addOnName)
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
})
})

View File

@@ -207,8 +207,8 @@ func (t *Tester) CreateKlusterlet(name, clusterName, agentNamespace string, mode
}
}
if mode == operatorapiv1.InstallModeDetached {
// create external-managed-kubeconfig, will use the same cluster to simulate the Detached mode.
if mode == operatorapiv1.InstallModeDetached || mode == operatorapiv1.InstallModeHosted {
// create external-managed-kubeconfig, will use the same cluster to simulate the Hosted mode.
secret.Namespace = agentNamespace
secret.Name = helpers.ExternalManagedKubeConfig
if _, err := t.KubeClient.CoreV1().Secrets(agentNamespace).Get(context.TODO(), secret.Name, metav1.GetOptions{}); err != nil {
@@ -253,7 +253,7 @@ func (t *Tester) CreatePureHostedKlusterlet(name, clusterName string) (*operator
},
ClusterName: clusterName,
DeployOption: operatorapiv1.KlusterletDeployOption{
Mode: operatorapiv1.InstallModeDetached,
Mode: operatorapiv1.InstallModeHosted,
},
},
}

View File

@@ -151,6 +151,7 @@ var _ = Describe("Create klusterlet CR", func() {
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
})
// TODO: remove this after the detached mode is not used in klusterlet
It("Create klusterlet CR in Detached mode", func() {
By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName))
_, err := t.CreateKlusterlet(klusterletName, clusterName, agentNamespace, operatorapiv1.InstallModeDetached)
@@ -189,6 +190,45 @@ var _ = Describe("Create klusterlet CR", func() {
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
})
It("Create klusterlet CR in Hosted mode", func() {
By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName))
_, err := t.CreateKlusterlet(klusterletName, clusterName, agentNamespace, operatorapiv1.InstallModeHosted)
Expect(err).ToNot(HaveOccurred())
By(fmt.Sprintf("waiting for the managed cluster %v to be created", clusterName))
Eventually(func() error {
_, err := t.GetCreatedManagedCluster(clusterName)
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("check klusterlet %s status", klusterletName))
Eventually(func() error {
err := t.checkKlusterletStatus(klusterletName, "HubConnectionDegraded", "BootstrapSecretFunctional,HubKubeConfigSecretMissing", metav1.ConditionTrue)
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("approve the created managed cluster %v", clusterName))
Eventually(func() error {
return t.ApproveCSR(clusterName)
}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
By(fmt.Sprintf("accept the created managed cluster %v", clusterName))
Eventually(func() error {
return t.AcceptsClient(clusterName)
}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
By(fmt.Sprintf("waiting for the managed cluster %v to be ready", clusterName))
Eventually(func() error {
return t.CheckManagedClusterStatus(clusterName)
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("check klusterlet %s status", klusterletName))
Eventually(func() error {
err := t.checkKlusterletStatus(klusterletName, "HubConnectionDegraded", "HubConnectionFunctional", metav1.ConditionFalse)
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
})
})
var _ = Describe("Delete klusterlet CR", func() {

View File

@@ -78,6 +78,7 @@ var _ = Describe("Create klusterlet and then create a configmap by manifestwork"
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(BeTrue())
})
// TODO: remove this after the detached mode is not used in klusterlet
It("Create configmap using manifestwork and then delete klusterlet in Detached mode", func() {
var err error
By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName))
@@ -125,4 +126,52 @@ var _ = Describe("Create klusterlet and then create a configmap by manifestwork"
return errors.IsNotFound(err)
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(BeTrue())
})
It("Create configmap using manifestwork and then delete klusterlet in Hosted mode", func() {
var err error
By(fmt.Sprintf("create klusterlet %v with managed cluster name %v", klusterletName, clusterName))
_, err = t.CreateKlusterlet(klusterletName, clusterName, agentNamespace, operatorapiv1.InstallModeHosted)
Expect(err).ToNot(HaveOccurred())
By(fmt.Sprintf("waiting for the managed cluster %v to be created", clusterName))
Eventually(func() error {
_, err = t.GetCreatedManagedCluster(clusterName)
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("approve the created managed cluster %v", clusterName))
Eventually(func() error {
return t.ApproveCSR(clusterName)
}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
By(fmt.Sprintf("accept the created managed cluster %v", clusterName))
Eventually(func() error {
return t.AcceptsClient(clusterName)
}, t.EventuallyTimeout, t.EventuallyInterval).Should(Succeed())
By(fmt.Sprintf("waiting for the managed cluster %v to be ready", clusterName))
Eventually(func() error {
return t.CheckManagedClusterStatus(clusterName)
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("create configmap %v/%v using manifestwork %v/%v", configMapNamespace,
configMapName, clusterName, workName))
_, err = t.CreateWorkOfConfigMap(workName, clusterName, configMapName, configMapNamespace)
Expect(err).ToNot(HaveOccurred())
By(fmt.Sprintf("waiting for configmap %v/%v to be created", configMapNamespace, configMapName))
Eventually(func() error {
_, err := t.KubeClient.CoreV1().ConfigMaps(configMapNamespace).
Get(context.TODO(), configMapName, metav1.GetOptions{})
return err
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(Succeed())
By(fmt.Sprintf("delete manifestwork %v/%v", clusterName, workName))
err = t.WorkClient.WorkV1().ManifestWorks(clusterName).Delete(context.Background(), workName, metav1.DeleteOptions{})
Expect(err).ToNot(HaveOccurred())
Eventually(func() bool {
_, err := t.WorkClient.WorkV1().ManifestWorks(clusterName).Get(context.Background(), workName, metav1.GetOptions{})
return errors.IsNotFound(err)
}, t.EventuallyTimeout*5, t.EventuallyInterval*5).Should(BeTrue())
})
})

View File

@@ -14,7 +14,7 @@ import (
"open-cluster-management.io/registration-operator/test/integration/util"
)
var _ = ginkgo.Describe("Klusterlet Detached mode", func() {
var _ = ginkgo.Describe("Klusterlet Hosted mode", func() {
var cancel context.CancelFunc
var klusterlet *operatorapiv1.Klusterlet
var klusterletNamespace string
@@ -43,7 +43,7 @@ var _ = ginkgo.Describe("Klusterlet Detached mode", func() {
},
ClusterName: "testcluster",
DeployOption: operatorapiv1.KlusterletDeployOption{
Mode: operatorapiv1.InstallModeDetached,
Mode: operatorapiv1.InstallModeHosted,
},
},
}