package e2e import ( "context" "encoding/json" "fmt" "io" "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" authv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" clusterapiv1 "open-cluster-management.io/api/cluster/v1" clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1" clusterapiv1beta2 "open-cluster-management.io/api/cluster/v1beta2" "open-cluster-management.io/ocm/test/framework" ) const ( placementDebuggerTestLabel = "placement-debugger-e2e-test" placementDebuggerServiceName = "cluster-manager-placement" placementDebuggerServicePort = 9443 placementDebuggerHubNamespace = "open-cluster-management-hub" placementDebuggerTimeout = 60 * time.Second placementDebuggerPollingInterval = 1 * time.Second ) type DebugResponse struct { Placement *clusterapiv1beta1.Placement `json:"placement,omitempty"` FilteredPipelineResults []interface{} `json:"filteredPipelineResults,omitempty"` PrioritizeResults []interface{} `json:"prioritizeResults,omitempty"` AggregatedScores []interface{} `json:"aggregatedScores,omitempty"` Error string `json:"error,omitempty"` } var _ = ginkgo.Describe("PlacementDebugger", ginkgo.Label("placement-debugger", "sanity-check"), func() { var namespace string var placementName string var clusterSetName string var clusterName string var serviceAccountName string var suffix string var token string ginkgo.BeforeEach(func() { suffix = rand.String(5) namespace = fmt.Sprintf("debugger-test-%s", suffix) placementName = fmt.Sprintf("test-placement-%s", suffix) clusterSetName = fmt.Sprintf("test-clusterset-%s", suffix) clusterName = fmt.Sprintf("test-cluster-%s", suffix) serviceAccountName = fmt.Sprintf("test-sa-%s", suffix) ginkgo.By(fmt.Sprintf("Creating test namespace: %s", namespace)) ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, Labels: map[string]string{ e2eTestLabel: placementDebuggerTestLabel, }, }, } _, err := hub.KubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating ManagedClusterSet: %s", clusterSetName)) clusterSet := &clusterapiv1beta2.ManagedClusterSet{ ObjectMeta: metav1.ObjectMeta{ Name: clusterSetName, Labels: map[string]string{ e2eTestLabel: placementDebuggerTestLabel, }, }, } _, err = hub.ClusterClient.ClusterV1beta2().ManagedClusterSets().Create(context.Background(), clusterSet, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating ManagedClusterSetBinding in namespace: %s", namespace)) binding := &clusterapiv1beta2.ManagedClusterSetBinding{ ObjectMeta: metav1.ObjectMeta{ Name: clusterSetName, Namespace: namespace, }, Spec: clusterapiv1beta2.ManagedClusterSetBindingSpec{ ClusterSet: clusterSetName, }, } _, err = hub.ClusterClient.ClusterV1beta2().ManagedClusterSetBindings(namespace).Create(context.Background(), binding, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating ManagedCluster: %s", clusterName)) cluster := &clusterapiv1.ManagedCluster{ ObjectMeta: metav1.ObjectMeta{ Name: clusterName, Labels: map[string]string{ e2eTestLabel: placementDebuggerTestLabel, "cluster.open-cluster-management.io/clusterset": clusterSetName, "env": "test", }, }, Spec: clusterapiv1.ManagedClusterSpec{ HubAcceptsClient: true, }, } _, err = hub.ClusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating Placement: %s", placementName)) placement := &clusterapiv1beta1.Placement{ ObjectMeta: metav1.ObjectMeta{ Name: placementName, Namespace: namespace, }, Spec: clusterapiv1beta1.PlacementSpec{ ClusterSets: []string{clusterSetName}, Tolerations: []clusterapiv1beta1.Toleration{ { Key: "cluster.open-cluster-management.io/unreachable", Operator: clusterapiv1beta1.TolerationOpExists, }, { Key: "cluster.open-cluster-management.io/unavailable", Operator: clusterapiv1beta1.TolerationOpExists, }, }, }, } _, err = hub.ClusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By(fmt.Sprintf("Creating ServiceAccount: %s", serviceAccountName)) sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: serviceAccountName, Namespace: namespace, }, } _, err = hub.KubeClient.CoreV1().ServiceAccounts(namespace).Create(context.Background(), sa, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By("Creating Role with placement permissions") role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: "placement-reader", Namespace: namespace, }, Rules: []rbacv1.PolicyRule{ { APIGroups: []string{"cluster.open-cluster-management.io"}, Resources: []string{"placements"}, Verbs: []string{"create", "get"}, }, }, } _, err = hub.KubeClient.RbacV1().Roles(namespace).Create(context.Background(), role, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By("Creating RoleBinding") roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: "placement-reader-binding", Namespace: namespace, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "Role", Name: "placement-reader", }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", Name: serviceAccountName, Namespace: namespace, }, }, } _, err = hub.KubeClient.RbacV1().RoleBindings(namespace).Create(context.Background(), roleBinding, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By("Creating ClusterRole for debug path access") clusterRoleName := fmt.Sprintf("debug-access-%s", namespace) clusterRole := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: clusterRoleName, Labels: map[string]string{ e2eTestLabel: placementDebuggerTestLabel, }, }, Rules: []rbacv1.PolicyRule{ { NonResourceURLs: []string{"/debug/placements/*"}, Verbs: []string{"get", "post"}, }, }, } _, err = hub.KubeClient.RbacV1().ClusterRoles().Create(context.Background(), clusterRole, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By("Creating ClusterRoleBinding") clusterRoleBinding := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("debug-access-%s-binding", namespace), Labels: map[string]string{ e2eTestLabel: placementDebuggerTestLabel, }, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: clusterRoleName, }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", Name: serviceAccountName, Namespace: namespace, }, }, } _, err = hub.KubeClient.RbacV1().ClusterRoleBindings().Create(context.Background(), clusterRoleBinding, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.By("Creating ServiceAccount token") tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ ExpirationSeconds: func(i int64) *int64 { return &i }(3600), }, } tokenResp, err := hub.KubeClient.CoreV1().ServiceAccounts(namespace).CreateToken( context.Background(), serviceAccountName, tokenRequest, metav1.CreateOptions{}, ) gomega.Expect(err).ToNot(gomega.HaveOccurred()) token = tokenResp.Status.Token gomega.Expect(token).ToNot(gomega.BeEmpty()) // Wait a bit for RBAC to propagate time.Sleep(5 * time.Second) }) ginkgo.AfterEach(func() { ginkgo.By("Cleaning up test resources") // Delete namespace err := hub.KubeClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { ginkgo.GinkgoLogr.Error(err, "Failed to delete namespace", "namespace", namespace) } // Delete ManagedCluster err = hub.ClusterClient.ClusterV1().ManagedClusters().Delete(context.Background(), clusterName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { ginkgo.GinkgoLogr.Error(err, "Failed to delete ManagedCluster", "cluster", clusterName) } // Delete ManagedClusterSet err = hub.ClusterClient.ClusterV1beta2().ManagedClusterSets().Delete(context.Background(), clusterSetName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { ginkgo.GinkgoLogr.Error(err, "Failed to delete ManagedClusterSet", "clusterSet", clusterSetName) } // Delete ClusterRole and ClusterRoleBinding clusterRoleName := fmt.Sprintf("debug-access-%s", namespace) err = hub.KubeClient.RbacV1().ClusterRoles().Delete(context.Background(), clusterRoleName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { ginkgo.GinkgoLogr.Error(err, "Failed to delete ClusterRole", "clusterRole", clusterRoleName) } clusterRoleBindingName := fmt.Sprintf("debug-access-%s-binding", namespace) err = hub.KubeClient.RbacV1().ClusterRoleBindings().Delete(context.Background(), clusterRoleBindingName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { ginkgo.GinkgoLogr.Error(err, "Failed to delete ClusterRoleBinding", "clusterRoleBinding", clusterRoleBindingName) } }) ginkgo.It("Should debug placement with GET method using curl pod", func() { ginkgo.By("Creating a curl pod to test the debugger service") debugPath := fmt.Sprintf("/debug/placements/%s/%s", namespace, placementName) response := callDebugServiceWithPod(hub, namespace, "GET", debugPath, token, nil) ginkgo.By("Validating GET response") gomega.Expect(response.Placement).ToNot(gomega.BeNil()) gomega.Expect(response.Placement.Name).To(gomega.Equal(placementName)) gomega.Expect(response.Placement.Namespace).To(gomega.Equal(namespace)) gomega.Expect(response.FilteredPipelineResults).ToNot(gomega.BeNil()) gomega.Expect(response.Error).To(gomega.BeEmpty()) }) ginkgo.It("Should debug placement with POST method using curl pod", func() { ginkgo.By("Creating a curl pod to test the debugger service") placementPayload := &clusterapiv1beta1.Placement{ TypeMeta: metav1.TypeMeta{ APIVersion: "cluster.open-cluster-management.io/v1beta1", Kind: "Placement", }, ObjectMeta: metav1.ObjectMeta{ Name: "test-post-placement", Namespace: namespace, }, Spec: clusterapiv1beta1.PlacementSpec{ ClusterSets: []string{clusterSetName}, Tolerations: []clusterapiv1beta1.Toleration{ { Key: "cluster.open-cluster-management.io/unreachable", Operator: clusterapiv1beta1.TolerationOpExists, }, { Key: "cluster.open-cluster-management.io/unavailable", Operator: clusterapiv1beta1.TolerationOpExists, }, }, }, } debugPath := "/debug/placements/" response := callDebugServiceWithPod(hub, namespace, "POST", debugPath, token, placementPayload) ginkgo.By("Validating POST response") gomega.Expect(response.Placement).ToNot(gomega.BeNil()) gomega.Expect(response.Placement.Name).To(gomega.Equal("test-post-placement")) gomega.Expect(response.Placement.Namespace).To(gomega.Equal(namespace)) gomega.Expect(response.FilteredPipelineResults).ToNot(gomega.BeNil()) gomega.Expect(response.Error).To(gomega.BeEmpty()) }) }) // callDebugServiceWithPod calls the placement debugger service using a curl pod func callDebugServiceWithPod(hub *framework.Hub, namespace, method, path, token string, payload interface{}) *DebugResponse { serviceURL := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d%s", placementDebuggerServiceName, placementDebuggerHubNamespace, placementDebuggerServicePort, path, ) podName := fmt.Sprintf("curl-test-%s", rand.String(5)) // Build curl command curlCmd := []string{ "curl", "-sk", "-H", fmt.Sprintf("Authorization: Bearer %s", token), } if method == "POST" && payload != nil { jsonData, err := json.Marshal(payload) gomega.Expect(err).ToNot(gomega.HaveOccurred()) curlCmd = append(curlCmd, "-X", "POST", "-H", "Content-Type: application/json", "-d", string(jsonData), ) } curlCmd = append(curlCmd, serviceURL) // Create a pod with curl pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: namespace, }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{ { Name: "curl", Image: "curlimages/curl:latest", Command: curlCmd, }, }, }, } ginkgo.By(fmt.Sprintf("Creating curl pod %s in namespace %s", podName, namespace)) _, err := hub.KubeClient.CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{}) gomega.Expect(err).ToNot(gomega.HaveOccurred()) // Wait for pod to complete ginkgo.By("Waiting for curl pod to complete") gomega.Eventually(func() corev1.PodPhase { p, err := hub.KubeClient.CoreV1().Pods(namespace).Get(context.Background(), podName, metav1.GetOptions{}) if err != nil { return corev1.PodUnknown } return p.Status.Phase }, placementDebuggerTimeout, placementDebuggerPollingInterval).Should(gomega.Or( gomega.Equal(corev1.PodSucceeded), gomega.Equal(corev1.PodFailed), )) // Get pod logs ginkgo.By("Retrieving curl pod logs") req := hub.KubeClient.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{}) logs, err := req.Stream(context.Background()) gomega.Expect(err).ToNot(gomega.HaveOccurred()) defer logs.Close() respBody, err := io.ReadAll(logs) gomega.Expect(err).ToNot(gomega.HaveOccurred()) ginkgo.GinkgoLogr.Info("Received response from debugger", "response", string(respBody)) // Clean up the pod err = hub.KubeClient.CoreV1().Pods(namespace).Delete(context.Background(), podName, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { ginkgo.GinkgoLogr.Error(err, "Failed to delete curl pod", "pod", podName) } var debugResp DebugResponse err = json.Unmarshal(respBody, &debugResp) gomega.Expect(err).ToNot(gomega.HaveOccurred()) return &debugResp }