diff --git a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go index 9b3ede352..3cad81bcc 100644 --- a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go +++ b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller.go @@ -3,7 +3,12 @@ package statuscontroller import ( "context" "fmt" + "io/ioutil" + "os" + "path" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -13,6 +18,7 @@ import ( "k8s.io/client-go/kubernetes" appslister "k8s.io/client-go/listers/apps/v1" corelister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" "github.com/openshift/library-go/pkg/controller/factory" @@ -99,7 +105,7 @@ func (k *klusterletStatusController) sync(ctx context.Context, controllerContext } // Check if bootstrap secret exists - _, err = k.kubeClient.CoreV1().Secrets(klusterletNS).Get(ctx, helpers.BootstrapHubKubeConfigSecret, metav1.GetOptions{}) + bootstrapSecret, err := k.kubeClient.CoreV1().Secrets(klusterletNS).Get(ctx, helpers.BootstrapHubKubeConfigSecret, metav1.GetOptions{}) if err != nil { registrationDegradedCondition.Message = fmt.Sprintf("Failed to get bootstrap secret %q %q: %v", klusterletNS, helpers.BootstrapHubKubeConfigSecret, err) registrationDegradedCondition.Status = metav1.ConditionTrue @@ -109,7 +115,39 @@ func (k *klusterletStatusController) sync(ctx context.Context, controllerContext ) return err } - // TODO verify if bootstrap secret works + + // Check if bootstrap secret works by building kube client + bootstrapClient, err := buildKubeClientWithSecret(bootstrapSecret) + if err != nil { + registrationDegradedCondition.Message = fmt.Sprintf("Failed to build kube client with bootstrap secret %q %q: %v", klusterletNS, helpers.BootstrapHubKubeConfigSecret, err) + registrationDegradedCondition.Status = metav1.ConditionTrue + registrationDegradedCondition.Reason = "BootstrapSecretError" + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(registrationDegradedCondition), + ) + return err + } + + // Check the bootstrap client permissions by creating SelfSubjectAccessReviews + allowed, failedReview, err := createSelfSubjectAccessReviews(ctx, bootstrapClient, getBootstrapSelfSubjectAccessReviews()) + if err != nil { + registrationDegradedCondition.Message = fmt.Sprintf("Failed to create %+v with bootstrap secret %q %q: %v", failedReview, klusterletNS, helpers.BootstrapHubKubeConfigSecret, err) + registrationDegradedCondition.Status = metav1.ConditionTrue + registrationDegradedCondition.Reason = "BootstrapSecretError" + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(registrationDegradedCondition), + ) + return err + } + if !allowed { + registrationDegradedCondition.Message = fmt.Sprintf("Operation for resource %+v is not allowed with bootstrap secret %q %q", failedReview.Spec.ResourceAttributes, klusterletNS, helpers.BootstrapHubKubeConfigSecret) + registrationDegradedCondition.Status = metav1.ConditionTrue + registrationDegradedCondition.Reason = "BootstrapSecretUnauthorized" + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(registrationDegradedCondition), + ) + return err + } // Check if hub kubeconfig secret exists hubConfigSecret, err := k.kubeClient.CoreV1().Secrets(klusterletNS).Get(ctx, helpers.HubKubeConfigSecret, metav1.GetOptions{}) @@ -130,9 +168,9 @@ func (k *klusterletStatusController) sync(ctx context.Context, controllerContext } // If cluster name is empty, read cluster name from hub config secret - if klusterlet.Spec.ClusterName == "" { - clusterName := hubConfigSecret.Data["cluster-name"] - if clusterName == nil { + clusterName := klusterlet.Spec.ClusterName + if clusterName == "" { + if hubConfigSecret.Data["cluster-name"] == nil { registrationDegradedCondition.Message = fmt.Sprintf( "Failed to get cluster name from `kubectl get secret -n %q %q -ojsonpath='{.data.cluster-name}`. This is set by the klusterlet registration deployment.", hubConfigSecret.Namespace, hubConfigSecret.Name) registrationDegradedCondition.Status = metav1.ConditionTrue @@ -148,6 +186,7 @@ func (k *klusterletStatusController) sync(ctx context.Context, controllerContext ) return err } + clusterName = string(hubConfigSecret.Data["cluster-name"]) } // If hub kubeconfig does not exist, return err. @@ -167,7 +206,65 @@ func (k *klusterletStatusController) sync(ctx context.Context, controllerContext ) return err } - // TODO it is possible to verify the kubeconfig actually works. + + // Check if hub config secret works by building kube client with its kubeconfig + hubClient, err := buildKubeClientWithSecret(hubConfigSecret) + if err != nil { + registrationDegradedCondition.Message = fmt.Sprintf("Failed to build kube client with hub config secret %q %q: %v", hubConfigSecret.Namespace, hubConfigSecret.Name, err) + registrationDegradedCondition.Status = metav1.ConditionTrue + registrationDegradedCondition.Reason = "HubConfigSecretError" + // Work condition will be the same as registration + workDegradedCondition.Message = registrationDegradedCondition.Message + workDegradedCondition.Status = registrationDegradedCondition.Status + workDegradedCondition.Reason = registrationDegradedCondition.Reason + + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(registrationDegradedCondition), + helpers.UpdateKlusterletConditionFn(workDegradedCondition), + ) + return err + } + + // Check the hub client (registration and work) permissions by creating SelfSubjectAccessReviews + allowed, failedReview, err = createSelfSubjectAccessReviews(ctx, hubClient, getRegistrationSelfSubjectAccessReviews(clusterName)) + if err != nil { + registrationDegradedCondition.Message = fmt.Sprintf("Failed to create %+v with hub config secret %q %q: %v", failedReview, klusterletNS, helpers.BootstrapHubKubeConfigSecret, err) + registrationDegradedCondition.Status = metav1.ConditionTrue + registrationDegradedCondition.Reason = "HubConfigSecretError" + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(registrationDegradedCondition), + ) + return err + } + if !allowed { + registrationDegradedCondition.Message = fmt.Sprintf("Operation for resource %+v is not allowed with hub config secret %q %q", failedReview.Spec.ResourceAttributes, klusterletNS, helpers.BootstrapHubKubeConfigSecret) + registrationDegradedCondition.Status = metav1.ConditionTrue + registrationDegradedCondition.Reason = "HubConfigSecretUnauthorized" + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(registrationDegradedCondition), + ) + return err + } + + allowed, failedReview, err = createSelfSubjectAccessReviews(ctx, hubClient, getWorkSelfSubjectAccessReviews(clusterName)) + if err != nil { + workDegradedCondition.Message = fmt.Sprintf("Failed to create %+v with hub config secret %q %q: %v", failedReview, klusterletNS, helpers.BootstrapHubKubeConfigSecret, err) + workDegradedCondition.Status = metav1.ConditionTrue + workDegradedCondition.Reason = "HubConfigSecretError" + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(workDegradedCondition), + ) + return err + } + if !allowed { + workDegradedCondition.Message = fmt.Sprintf("Operation for resource %+v is not allowed with hub config secret %q %q", failedReview.Spec.ResourceAttributes, klusterletNS, helpers.BootstrapHubKubeConfigSecret) + workDegradedCondition.Status = metav1.ConditionTrue + workDegradedCondition.Reason = "HubConfigSecretUnauthorized" + _, _, err := helpers.UpdateKlusterletStatus(ctx, k.klusterletClient, klusterletName, + helpers.UpdateKlusterletConditionFn(workDegradedCondition), + ) + return err + } // Check deployment status registrationDeploymentName := fmt.Sprintf("%s-registration-agent", klusterlet.Name) @@ -200,3 +297,144 @@ func (k *klusterletStatusController) sync(ctx context.Context, controllerContext ) return nil } + +func buildKubeClientWithSecret(secret *corev1.Secret) (kubernetes.Interface, error) { + tempdir, err := ioutil.TempDir("", "kube") + if err != nil { + return nil, err + } + defer os.RemoveAll(tempdir) + + for key, data := range secret.Data { + if err := ioutil.WriteFile(path.Join(tempdir, key), data, 0644); err != nil { + return nil, err + } + } + restConfig, err := clientcmd.BuildConfigFromFlags("", path.Join(tempdir, "kubeconfig")) + if err != nil { + return nil, err + } + return kubernetes.NewForConfig(restConfig) +} + +func createSelfSubjectAccessReviews( + ctx context.Context, + kubeClient kubernetes.Interface, + selfSubjectAccessReviews []authorizationv1.SelfSubjectAccessReview) (bool, *authorizationv1.SelfSubjectAccessReview, error) { + for i := range selfSubjectAccessReviews { + subjectAccessReview := selfSubjectAccessReviews[i] + ssar, err := kubeClient.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, &subjectAccessReview, metav1.CreateOptions{}) + if err != nil { + return false, &subjectAccessReview, err + } + if !ssar.Status.Allowed { + return false, &subjectAccessReview, nil + } + } + return true, nil, nil +} + +func getBootstrapSelfSubjectAccessReviews() []authorizationv1.SelfSubjectAccessReview { + reviews := []authorizationv1.SelfSubjectAccessReview{} + clusterResource := authorizationv1.ResourceAttributes{ + Group: "cluster.open-cluster-management.io", + Resource: "managedclusters", + } + reviews = append(reviews, generateSelfSubjectAccessReviews(clusterResource, "create", "get")...) + + certResource := authorizationv1.ResourceAttributes{ + Group: "certificates.k8s.io", + Resource: "certificatesigningrequests", + } + return append(reviews, generateSelfSubjectAccessReviews(certResource, "create", "get", "list", "watch")...) +} + +func getRegistrationSelfSubjectAccessReviews(clusterName string) []authorizationv1.SelfSubjectAccessReview { + reviews := []authorizationv1.SelfSubjectAccessReview{} + certResource := authorizationv1.ResourceAttributes{ + Group: "certificates.k8s.io", + Resource: "certificatesigningrequests", + } + reviews = append(reviews, generateSelfSubjectAccessReviews(certResource, "get", "list", "watch")...) + + clusterResource := authorizationv1.ResourceAttributes{ + Group: "cluster.open-cluster-management.io", + Resource: "managedclusters", + Name: clusterName, + } + reviews = append(reviews, generateSelfSubjectAccessReviews(clusterResource, "get", "list", "update", "watch")...) + + clusterStatusResource := authorizationv1.ResourceAttributes{ + Group: "cluster.open-cluster-management.io", + Resource: "managedclusters", + Subresource: "status", + Name: clusterName, + } + reviews = append(reviews, generateSelfSubjectAccessReviews(clusterStatusResource, "patch", "update")...) + + clusterCertResource := authorizationv1.ResourceAttributes{ + Group: "register.open-cluster-management.io", + Resource: "managedclusters", + Subresource: "clientcertificates", + } + reviews = append(reviews, generateSelfSubjectAccessReviews(clusterCertResource, "renew")...) + + leaseResource := authorizationv1.ResourceAttributes{ + Group: "coordination.k8s.io", + Resource: "leases", + Name: fmt.Sprintf("cluster-lease-%s", clusterName), + Namespace: clusterName, + } + return append(reviews, generateSelfSubjectAccessReviews(leaseResource, "get", "update")...) +} + +func getWorkSelfSubjectAccessReviews(clusterName string) []authorizationv1.SelfSubjectAccessReview { + reviews := []authorizationv1.SelfSubjectAccessReview{} + eventResource := authorizationv1.ResourceAttributes{ + Resource: "events", + Namespace: clusterName, + } + reviews = append(reviews, generateSelfSubjectAccessReviews(eventResource, "create", "patch", "update")...) + + eventResource = authorizationv1.ResourceAttributes{ + Group: "events.k8s.io", + Resource: "events", + Namespace: clusterName, + } + reviews = append(reviews, generateSelfSubjectAccessReviews(eventResource, "create", "patch", "update")...) + + workResource := authorizationv1.ResourceAttributes{ + Group: "work.open-cluster-management.io", + Resource: "manifestworks", + Namespace: clusterName, + } + reviews = append(reviews, generateSelfSubjectAccessReviews(workResource, "get", "list", "watch", "update")...) + + workStatusResource := authorizationv1.ResourceAttributes{ + Group: "work.open-cluster-management.io", + Resource: "manifestworks", + Subresource: "status", + Namespace: clusterName, + } + reviews = append(reviews, generateSelfSubjectAccessReviews(workStatusResource, "patch", "update")...) + return reviews +} + +func generateSelfSubjectAccessReviews(resource authorizationv1.ResourceAttributes, verbs ...string) []authorizationv1.SelfSubjectAccessReview { + reviews := []authorizationv1.SelfSubjectAccessReview{} + for _, verb := range verbs { + reviews = append(reviews, authorizationv1.SelfSubjectAccessReview{ + Spec: authorizationv1.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Group: resource.Group, + Resource: resource.Resource, + Subresource: resource.Subresource, + Name: resource.Name, + Namespace: resource.Namespace, + Verb: verb, + }, + }, + }) + } + return reviews +} diff --git a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go index 5754794e8..f9e572a4d 100644 --- a/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go +++ b/pkg/operators/klusterlet/controllers/statuscontroller/klusterlet_status_controller_test.go @@ -1,16 +1,24 @@ package statuscontroller import ( + "context" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" "testing" "time" appsv1 "k8s.io/api/apps/v1" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" kubeinformers "k8s.io/client-go/informers" fakekube "k8s.io/client-go/kubernetes/fake" clienttesting "k8s.io/client-go/testing" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" fakeoperatorclient "github.com/open-cluster-management/api/client/operator/clientset/versioned/fake" operatorinformers "github.com/open-cluster-management/api/client/operator/informers/externalversions" @@ -24,6 +32,12 @@ type testController struct { operatorClient *fakeoperatorclient.Clientset } +type serverResponse struct { + allowToOperateManagedClusters bool + allowToOperateManagedClusterStatus bool + allowToOperateManifestWorks bool +} + func newSecret(name, namespace string) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -34,9 +48,23 @@ func newSecret(name, namespace string) *corev1.Secret { } } -func newSecretWithKubeConfig(name, namespace string) *corev1.Secret { +func newKubeConfig(host string) []byte { + configData, _ := runtime.Encode(clientcmdlatest.Codec, &clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": { + Server: host, + InsecureSkipTLSVerify: true, + }}, + Contexts: map[string]*clientcmdapi.Context{"default-context": { + Cluster: "default-cluster", + }}, + CurrentContext: "default-context", + }) + return configData +} + +func newSecretWithKubeConfig(name, namespace string, kubeConfig []byte) *corev1.Secret { secret := newSecret(name, namespace) - secret.Data["kubeconfig"] = []byte("kubeconfig") + secret.Data["kubeconfig"] = kubeConfig return secret } @@ -94,11 +122,48 @@ func newTestController(klusterlet *operatorapiv1.Klusterlet, objects ...runtime. } func TestSync(t *testing.T) { + response := &serverResponse{} + apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/apis/authorization.k8s.io/v1/selfsubjectaccessreviews" { + w.WriteHeader(http.StatusNotFound) + return + } + + data, err := ioutil.ReadAll(req.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + ssar := &authorizationv1.SelfSubjectAccessReview{} + json.Unmarshal(data, ssar) + if ssar.Spec.ResourceAttributes.Resource == "managedclusters" { + if ssar.Spec.ResourceAttributes.Subresource == "status" { + ssar.Status.Allowed = response.allowToOperateManagedClusterStatus + } else { + ssar.Status.Allowed = response.allowToOperateManagedClusters + } + } else if ssar.Spec.ResourceAttributes.Resource == "manifestworks" { + ssar.Status.Allowed = response.allowToOperateManifestWorks + } else { + ssar.Status.Allowed = true + } + + w.Header().Set("Content-type", "application/json") + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(ssar) + })) + defer apiServer.Close() + + apiServerHost := apiServer.URL + cases := []struct { - name string - object []runtime.Object - klusterlet *operatorapiv1.Klusterlet - expectedConditions []operatorapiv1.StatusCondition + name string + object []runtime.Object + klusterlet *operatorapiv1.Klusterlet + allowToOperateManagedClusters bool + allowToOperateManagedClusterStatus bool + allowToOperateManifestWorks bool + expectedConditions []operatorapiv1.StatusCondition }{ { name: "No bootstrap secret", @@ -109,41 +174,113 @@ func TestSync(t *testing.T) { }, }, { - name: "No hubconfig secret", - object: []runtime.Object{newSecret(helpers.BootstrapHubKubeConfigSecret, "test")}, + name: "Bad bootstrap secret", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", []byte("badsecret")), + }, klusterlet: newKlusterlet("testklusterlet", "test", ""), + expectedConditions: []operatorapiv1.StatusCondition{ + testinghelper.NamedCondition(klusterletRegistrationDegraded, "BootstrapSecretError", metav1.ConditionTrue), + }, + }, + { + name: "Unauthorized bootstrap secret", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + }, + klusterlet: newKlusterlet("testklusterlet", "test", ""), + expectedConditions: []operatorapiv1.StatusCondition{ + testinghelper.NamedCondition(klusterletRegistrationDegraded, "BootstrapSecretUnauthorized", metav1.ConditionTrue), + }, + }, + { + name: "No hubconfig secret", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + }, + klusterlet: newKlusterlet("testklusterlet", "test", ""), + allowToOperateManagedClusters: true, expectedConditions: []operatorapiv1.StatusCondition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "HubKubeConfigSecretMissing", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "HubKubeConfigSecretMissing", metav1.ConditionTrue), }, }, { - name: "No cluster name secret", - object: []runtime.Object{newSecret(helpers.BootstrapHubKubeConfigSecret, "test"), newSecret(helpers.HubKubeConfigSecret, "test")}, - klusterlet: newKlusterlet("testklusterlet", "test", ""), + name: "No cluster name secret", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + newSecret(helpers.HubKubeConfigSecret, "test"), + }, + allowToOperateManagedClusters: true, + klusterlet: newKlusterlet("testklusterlet", "test", ""), expectedConditions: []operatorapiv1.StatusCondition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "ClusterNameMissing", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "ClusterNameMissing", metav1.ConditionTrue), }, }, { - name: "No kubeconfig secret", - object: []runtime.Object{newSecret(helpers.BootstrapHubKubeConfigSecret, "test"), newSecret(helpers.HubKubeConfigSecret, "test")}, - klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), + name: "No kubeconfig secret", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + newSecret(helpers.HubKubeConfigSecret, "test"), + }, + allowToOperateManagedClusters: true, + klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), expectedConditions: []operatorapiv1.StatusCondition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "KubeConfigMissing", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "KubeConfigMissing", metav1.ConditionTrue), }, }, + { + name: "Bad hub config secret", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + newSecretWithKubeConfig(helpers.HubKubeConfigSecret, "test", []byte("badkubeconfig")), + }, + allowToOperateManagedClusters: true, + klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), + expectedConditions: []operatorapiv1.StatusCondition{ + testinghelper.NamedCondition(klusterletRegistrationDegraded, "HubConfigSecretError", metav1.ConditionTrue), + testinghelper.NamedCondition(klusterletWorKDegraded, "HubConfigSecretError", metav1.ConditionTrue), + }, + }, + { + name: "Unauthorized hub config secret (registration)", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + newSecretWithKubeConfig(helpers.HubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + }, + allowToOperateManagedClusters: true, + klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), + expectedConditions: []operatorapiv1.StatusCondition{ + testinghelper.NamedCondition(klusterletRegistrationDegraded, "HubConfigSecretUnauthorized", metav1.ConditionTrue), + }, + }, + { + name: "Unauthorized hub config secret (work)", + object: []runtime.Object{ + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + newSecretWithKubeConfig(helpers.HubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + }, + allowToOperateManagedClusters: true, + allowToOperateManagedClusterStatus: true, + klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), + expectedConditions: []operatorapiv1.StatusCondition{ + testinghelper.NamedCondition(klusterletWorKDegraded, "HubConfigSecretUnauthorized", metav1.ConditionTrue), + }, + }, { name: "Unavailable pod in deployments", object: []runtime.Object{ - newSecret(helpers.BootstrapHubKubeConfigSecret, "test"), - newSecretWithKubeConfig(helpers.HubKubeConfigSecret, "test"), + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + newSecretWithKubeConfig(helpers.HubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), newDeployment("testklusterlet-registration-agent", "test", 3, 0), newDeployment("testklusterlet-work-agent", "test", 3, 0), }, - klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), + allowToOperateManagedClusters: true, + allowToOperateManagedClusterStatus: true, + allowToOperateManifestWorks: true, + klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), expectedConditions: []operatorapiv1.StatusCondition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "UnavailableRegistrationPod", metav1.ConditionTrue), testinghelper.NamedCondition(klusterletWorKDegraded, "UnavailableWorkPod", metav1.ConditionTrue), @@ -152,12 +289,15 @@ func TestSync(t *testing.T) { { name: "Operator functional", object: []runtime.Object{ - newSecret(helpers.BootstrapHubKubeConfigSecret, "test"), - newSecretWithKubeConfig(helpers.HubKubeConfigSecret, "test"), + newSecretWithKubeConfig(helpers.BootstrapHubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), + newSecretWithKubeConfig(helpers.HubKubeConfigSecret, "test", newKubeConfig(apiServerHost)), newDeployment("testklusterlet-registration-agent", "test", 3, 3), newDeployment("testklusterlet-work-agent", "test", 3, 3), }, - klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), + allowToOperateManagedClusters: true, + allowToOperateManagedClusterStatus: true, + allowToOperateManifestWorks: true, + klusterlet: newKlusterlet("testklusterlet", "test", "cluster1"), expectedConditions: []operatorapiv1.StatusCondition{ testinghelper.NamedCondition(klusterletRegistrationDegraded, "RegistrationFunctional", metav1.ConditionFalse), testinghelper.NamedCondition(klusterletWorKDegraded, "WorkFunctional", metav1.ConditionFalse), @@ -169,18 +309,21 @@ func TestSync(t *testing.T) { t.Run(c.name, func(t *testing.T) { controller := newTestController(c.klusterlet, c.object...) syncContext := testinghelper.NewFakeSyncContext(t, c.klusterlet.Name) - err := controller.controller.sync(nil, syncContext) + + response.allowToOperateManagedClusters = c.allowToOperateManagedClusters + response.allowToOperateManagedClusterStatus = c.allowToOperateManagedClusterStatus + response.allowToOperateManifestWorks = c.allowToOperateManifestWorks + + err := controller.controller.sync(context.TODO(), syncContext) if err != nil { t.Errorf("Expected no error when update status: %v", err) } - operatorActions := controller.operatorClient.Actions() testinghelper.AssertEqualNumber(t, len(operatorActions), 2) testinghelper.AssertGet(t, operatorActions[0], "operator.open-cluster-management.io", "v1", "klusterlets") testinghelper.AssertAction(t, operatorActions[1], "update") - testinghelper.AssertOnlyConditions( - t, operatorActions[1].(clienttesting.UpdateActionImpl).Object, c.expectedConditions...) + testinghelper.AssertOnlyConditions(t, operatorActions[1].(clienttesting.UpdateActionImpl).Object, c.expectedConditions...) }) } } diff --git a/test/integration/integration_suite_test.go b/test/integration/integration_suite_test.go index f5992b5ba..5c6d42a75 100644 --- a/test/integration/integration_suite_test.go +++ b/test/integration/integration_suite_test.go @@ -53,6 +53,7 @@ var _ = ginkgo.BeforeSuite(func(done ginkgo.Done) { }, } cfg, err := testEnv.Start() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) gomega.Expect(cfg).ToNot(gomega.BeNil()) diff --git a/test/integration/klusterlet_test.go b/test/integration/klusterlet_test.go index dfa45777c..e50b3a4c6 100644 --- a/test/integration/klusterlet_test.go +++ b/test/integration/klusterlet_test.go @@ -371,14 +371,14 @@ var _ = ginkgo.Describe("Klusterlet", func() { util.AssertKlusterletCondition(klusterlet.Name, operatorClient, "KlusterletRegistrationDegraded", "BootStrapSecretMissing", metav1.ConditionTrue) - // Create a dummy bootstrap secret + // Create a bootstrap secret and make sure the kubeconfig can work bootStrapSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: helpers.BootstrapHubKubeConfigSecret, Namespace: klusterletNamespace, }, Data: map[string][]byte{ - "kubeconfig": []byte("dummy"), + "kubeconfig": util.NewKubeConfig(restConfig.Host), }, } _, err = kubeClient.CoreV1().Secrets(klusterletNamespace).Create(context.Background(), bootStrapSecret, metav1.CreateOptions{}) @@ -390,9 +390,9 @@ var _ = ginkgo.Describe("Klusterlet", func() { hubSecret, err := kubeClient.CoreV1().Secrets(klusterletNamespace).Get(context.Background(), helpers.HubKubeConfigSecret, metav1.GetOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) - // Update hub secret + // Update hub secret and make sure the kubeconfig can work hubSecret.Data["cluster-name"] = []byte("testcluster") - hubSecret.Data["kubeconfig"] = []byte("dummy") + hubSecret.Data["kubeconfig"] = util.NewKubeConfig(restConfig.Host) _, err = kubeClient.CoreV1().Secrets(klusterletNamespace).Update(context.Background(), hubSecret, metav1.UpdateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) diff --git a/test/integration/util/util.go b/test/integration/util/util.go index e937b4f32..266040feb 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -7,6 +7,9 @@ import ( "github.com/openshift/library-go/pkg/operator/events" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + clientcmdlatest "k8s.io/client-go/tools/clientcmd/api/latest" operatorapiv1 "github.com/open-cluster-management/api/operator/v1" ) @@ -54,6 +57,7 @@ func (r *IntegrationTestEventRecorder) Shutdown() { func HasCondition(conditions []operatorapiv1.StatusCondition, expectedType, expectedReason string, expectedStatus metav1.ConditionStatus) bool { found := false for _, condition := range conditions { + fmt.Printf(">>> %v \n", condition) if condition.Type != expectedType { continue } @@ -72,3 +76,17 @@ func HasCondition(conditions []operatorapiv1.StatusCondition, expectedType, expe return found } + +func NewKubeConfig(host string) []byte { + configData, _ := runtime.Encode(clientcmdlatest.Codec, &clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{"test-cluster": { + Server: host, + InsecureSkipTLSVerify: true, + }}, + Contexts: map[string]*clientcmdapi.Context{"test-context": { + Cluster: "test-cluster", + }}, + CurrentContext: "test-context", + }) + return configData +}