From 1bbfd694087b8d0a8a3da030d5c646c3920e7ae1 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Tue, 21 Apr 2026 21:41:49 +0800 Subject: [PATCH] Fix flaky clustermanager integration test (#1493) Signed-off-by: Jian Qiu --- .../operator/clustermanager_hosted_test.go | 20 ++++++++++++ .../operator/clustermanager_test.go | 32 +++++++++++++++---- .../spokeagent_rebootstrap_test.go | 2 +- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/test/integration/operator/clustermanager_hosted_test.go b/test/integration/operator/clustermanager_hosted_test.go index ec8a8dc81..e34ebff4e 100644 --- a/test/integration/operator/clustermanager_hosted_test.go +++ b/test/integration/operator/clustermanager_hosted_test.go @@ -42,6 +42,26 @@ func updateDeploymentsStatusSuccess(kubeClient kubernetes.Interface, namespace s } } +func updateAllDeploymentsStatusSuccess(kubeClient kubernetes.Interface, namespace string) { + deployments, err := kubeClient.AppsV1().Deployments(namespace).List(context.Background(), metav1.ListOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + var names []string + for _, d := range deployments.Items { + names = append(names, d.Name) + } + updateDeploymentsStatusSuccess(kubeClient, namespace, names...) +} + +func updateAllDeploymentsStatusFail(kubeClient kubernetes.Interface, namespace string) { + deployments, err := kubeClient.AppsV1().Deployments(namespace).List(context.Background(), metav1.ListOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + var names []string + for _, d := range deployments.Items { + names = append(names, d.Name) + } + updateDeploymentsStatusFail(kubeClient, namespace, names...) +} + func updateDeploymentsStatusFail(kubeClient kubernetes.Interface, namespace string, deployments ...string) { for _, deploymentName := range deployments { gomega.Eventually(func() error { diff --git a/test/integration/operator/clustermanager_test.go b/test/integration/operator/clustermanager_test.go index 7a42c7975..ff5732f07 100644 --- a/test/integration/operator/clustermanager_test.go +++ b/test/integration/operator/clustermanager_test.go @@ -13,6 +13,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/rest" @@ -284,9 +285,7 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", ginkgo.Ordered, func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - updateDeploymentsStatusFail(kubeClient, hubNamespace, - hubRegistrationDeployment, hubPlacementDeployment, hubRegistrationWebhookDeployment, - hubWorkWebhookDeployment, hubWorkControllerDeployment, hubAddOnManagerDeployment, hubAddOnWebhookDeployment) + updateAllDeploymentsStatusFail(kubeClient, hubNamespace) // Check validating webhook registrationValidatingWebhook := "managedclustervalidators.admission.cluster.open-cluster-management.io" @@ -310,9 +309,27 @@ var _ = ginkgo.Describe("ClusterManager Default Mode", ginkgo.Ordered, func() { return nil }, eventuallyTimeout, eventuallyInterval).Should(gomega.BeNil()) - updateDeploymentsStatusSuccess(kubeClient, hubNamespace, - hubRegistrationDeployment, hubPlacementDeployment, hubRegistrationWebhookDeployment, - hubWorkWebhookDeployment, hubWorkControllerDeployment, hubAddOnManagerDeployment, hubAddOnWebhookDeployment) + updateAllDeploymentsStatusSuccess(kubeClient, hubNamespace) + + // Wait for ConditionProgressing to become False before checking webhooks. + // This ensures the operator has reconciled all deployment status updates + // and avoids a race condition where the webhook check starts before the + // operator sees all deployments as ready. + gomega.Eventually(func() error { + clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get( + context.Background(), clusterManagerName, metav1.GetOptions{}) + if err != nil { + return err + } + cond := meta.FindStatusCondition(clusterManager.Status.Conditions, operatorapiv1.ConditionProgressing) + if cond == nil { + return fmt.Errorf("ConditionProgressing not found") + } + if cond.Status != metav1.ConditionFalse { + return fmt.Errorf("ConditionProgressing is %s, waiting for False", cond.Status) + } + return nil + }, eventuallyTimeout*2, eventuallyInterval).Should(gomega.BeNil()) gomega.Eventually(func() error { _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get( @@ -1642,7 +1659,7 @@ var _ = ginkgo.Describe("ClusterManager TLS Profile", func() { context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - // Clean up PlacementConfiguration to avoid affecting other tests + // Clean up PlacementConfiguration and RegistrationConfiguration to avoid affecting other tests gomega.Eventually(func() error { clusterManager, err := operatorClient.OperatorV1().ClusterManagers().Get( context.Background(), clusterManagerName, metav1.GetOptions{}) @@ -1650,6 +1667,7 @@ var _ = ginkgo.Describe("ClusterManager TLS Profile", func() { return err } clusterManager.Spec.PlacementConfiguration = nil + clusterManager.Spec.RegistrationConfiguration = nil _, err = operatorClient.OperatorV1().ClusterManagers().Update( context.Background(), clusterManager, metav1.UpdateOptions{}) return err diff --git a/test/integration/registration/spokeagent_rebootstrap_test.go b/test/integration/registration/spokeagent_rebootstrap_test.go index 96ff52c12..a6b38b781 100644 --- a/test/integration/registration/spokeagent_rebootstrap_test.go +++ b/test/integration/registration/spokeagent_rebootstrap_test.go @@ -372,7 +372,7 @@ var _ = ginkgo.Describe("Rebootstrap", func() { case <-agentCtx.Done(): // restart agent fmt.Println("[agent-scheduler] - restart agent...") - agentCtx, stopAgent, _ = startAgent(spokeCtx, managedClusterName, hubKubeconfigDir, agentOptions) + agentCtx, stopAgent, agentConfig = startAgent(spokeCtx, managedClusterName, hubKubeconfigDir, agentOptions) case <-spokeCtx.Done(): // exit fmt.Println("[agent-scheduler] - shutting down...")