package framework import ( "context" "fmt" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" operatorapiv1 "open-cluster-management.io/api/operator/v1" "open-cluster-management.io/ocm/pkg/operator/helpers" ) // CreateAndApproveKlusterlet requires operations on both hub side and spoke side func CreateAndApproveKlusterlet( hub *Hub, spoke *Spoke, klusterletName, managedClusterName, klusterletNamespace string, mode operatorapiv1.InstallMode, bootstrapHubKubeConfigSecret *corev1.Secret, images Images, ) { // on the spoke side _, err := spoke.CreateKlusterlet( klusterletName, managedClusterName, klusterletNamespace, mode, bootstrapHubKubeConfigSecret, images, ) Expect(err).ToNot(HaveOccurred()) // on the hub side Eventually(func() error { _, err := hub.GetManagedCluster(managedClusterName) return err }).Should(Succeed()) Eventually(func() error { return hub.ApproveManagedClusterCSR(managedClusterName) }).Should(Succeed()) Eventually(func() error { return hub.AcceptManageCluster(managedClusterName) }).Should(Succeed()) Eventually(func() error { return hub.CheckManagedClusterStatus(managedClusterName) }).Should(Succeed()) } func (spoke *Spoke) CreateKlusterlet( name, clusterName, klusterletNamespace string, mode operatorapiv1.InstallMode, bootstrapHubKubeConfigSecret *corev1.Secret, images Images) (*operatorapiv1.Klusterlet, error) { if name == "" { return nil, fmt.Errorf("the name should not be null") } if klusterletNamespace == "" { klusterletNamespace = helpers.KlusterletDefaultNamespace } var klusterlet = &operatorapiv1.Klusterlet{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: operatorapiv1.KlusterletSpec{ RegistrationImagePullSpec: images.RegistrationImage, WorkImagePullSpec: images.WorkImage, ImagePullSpec: images.SingletonImage, ExternalServerURLs: []operatorapiv1.ServerURL{ { URL: "https://localhost", }, }, ClusterName: clusterName, Namespace: klusterletNamespace, DeployOption: operatorapiv1.KlusterletDeployOption{ Mode: mode, }, }, } agentNamespace := helpers.AgentNamespace(klusterlet) klog.Infof("klusterlet: %s/%s, \t mode: %v, \t agent namespace: %s", klusterlet.Name, klusterlet.Namespace, mode, agentNamespace) // create agentNamespace namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: agentNamespace, Annotations: map[string]string{ "workload.openshift.io/allowed": "management", }, }, } if _, err := spoke.KubeClient.CoreV1().Namespaces().Get(context.TODO(), agentNamespace, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { klog.Errorf("failed to get ns %v. %v", agentNamespace, err) return nil, err } if _, err := spoke.KubeClient.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}); err != nil { klog.Errorf("failed to create ns %v. %v", namespace, err) return nil, err } } // create bootstrap-hub-kubeconfig secret secret := bootstrapHubKubeConfigSecret.DeepCopy() if _, err := spoke.KubeClient.CoreV1().Secrets(agentNamespace).Get(context.TODO(), secret.Name, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { klog.Errorf("failed to get secret %v in ns %v. %v", secret.Name, agentNamespace, err) return nil, err } if _, err = spoke.KubeClient.CoreV1().Secrets(agentNamespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { klog.Errorf("failed to create secret %v in ns %v. %v", secret, agentNamespace, err) return nil, err } } if helpers.IsHosted(mode) { // create external-managed-kubeconfig, will use the same cluster to simulate the Hosted mode. secret.Namespace = agentNamespace secret.Name = helpers.ExternalManagedKubeConfig if _, err := spoke.KubeClient.CoreV1().Secrets(agentNamespace).Get(context.TODO(), secret.Name, metav1.GetOptions{}); err != nil { if !apierrors.IsNotFound(err) { klog.Errorf("failed to get secret %v in ns %v. %v", secret.Name, agentNamespace, err) return nil, err } if _, err = spoke.KubeClient.CoreV1().Secrets(agentNamespace).Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil { klog.Errorf("failed to create secret %v in ns %v. %v", secret, agentNamespace, err) return nil, err } } } // create klusterlet CR realKlusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Create(context.TODO(), klusterlet, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { klog.Errorf("failed to create klusterlet %v . %v", klusterlet.Name, err) return nil, err } return realKlusterlet, nil } func (spoke *Spoke) CreatePureHostedKlusterlet(name, clusterName string) (*operatorapiv1.Klusterlet, error) { if name == "" { return nil, fmt.Errorf("the name should not be null") } var klusterlet = &operatorapiv1.Klusterlet{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: operatorapiv1.KlusterletSpec{ RegistrationImagePullSpec: "quay.io/open-cluster-management/registration:latest", WorkImagePullSpec: "quay.io/open-cluster-management/work:latest", ExternalServerURLs: []operatorapiv1.ServerURL{ { URL: "https://localhost", }, }, ClusterName: clusterName, DeployOption: operatorapiv1.KlusterletDeployOption{ Mode: operatorapiv1.InstallModeHosted, }, }, } // create klusterlet CR realKlusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Create(context.TODO(), klusterlet, metav1.CreateOptions{}) if err != nil { klog.Errorf("failed to create klusterlet %v . %v", klusterlet.Name, err) return nil, err } return realKlusterlet, nil } func (spoke *Spoke) CheckKlusterletStatus(klusterletName, condType, reason string, status metav1.ConditionStatus) error { klusterlet, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get(context.TODO(), klusterletName, metav1.GetOptions{}) if err != nil { return err } cond := meta.FindStatusCondition(klusterlet.Status.Conditions, condType) if cond == nil { return fmt.Errorf("cannot find condition type %s", condType) } if cond.Reason != reason { return fmt.Errorf("condition reason is not matched, expect %s, got %s", reason, cond.Reason) } if cond.Status != status { return fmt.Errorf("condition status is not matched, expect %s, got %s", status, cond.Status) } return nil } // CleanKlusterletRelatedResources needs both hub side and spoke side operations func CleanKlusterletRelatedResources( hub *Hub, spoke *Spoke, klusterletName, managedClusterName string) { Expect(klusterletName).NotTo(Equal("")) // clean the klusterlet err := spoke.OperatorClient.OperatorV1().Klusterlets().Delete(context.TODO(), klusterletName, metav1.DeleteOptions{}) if apierrors.IsNotFound(err) { return } Expect(err).To(BeNil()) Eventually(func() error { _, err := spoke.OperatorClient.OperatorV1().Klusterlets().Get(context.TODO(), klusterletName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { klog.Infof("klusterlet %s deleted successfully", klusterletName) return nil } if err != nil { klog.Infof("get klusterlet %s error: %v", klusterletName, err) return err } return fmt.Errorf("klusterlet %s still exists", klusterletName) }).Should(Succeed()) // clean the managed clusters err = hub.ClusterClient.ClusterV1().ManagedClusters().Delete(context.TODO(), managedClusterName, metav1.DeleteOptions{}) if apierrors.IsNotFound(err) { return } Expect(err).To(BeNil()) Eventually(func() error { _, err := hub.ClusterClient.ClusterV1().ManagedClusters().Get(context.TODO(), managedClusterName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { klog.Infof("managed cluster %s deleted successfully", managedClusterName) return nil } if err != nil { klog.Infof("get managed cluster %s error: %v", klusterletName, err) return err } return fmt.Errorf("managed cluster %s still exists", managedClusterName) }).Should(Succeed()) }