diff --git a/test/e2e/e2e_duplicatepods_test.go b/test/e2e/e2e_duplicatepods_test.go index 3db674666..ff68dde34 100644 --- a/test/e2e/e2e_duplicatepods_test.go +++ b/test/e2e/e2e_duplicatepods_test.go @@ -192,32 +192,7 @@ func TestRemoveDuplicates(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, removeDuplicatesPolicy(tc.removeDuplicatesArgs, tc.evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - defer func() { - if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemoveDuplicates strategy var meetsExpectations bool diff --git a/test/e2e/e2e_evictioninbackground_test.go b/test/e2e/e2e_evictioninbackground_test.go index 47a6625cb..236bead2e 100644 --- a/test/e2e/e2e_evictioninbackground_test.go +++ b/test/e2e/e2e_evictioninbackground_test.go @@ -106,86 +106,51 @@ func formatContainerStatuses(pod *corev1.Pod) string { return strings.Join(parts, "; ") } -// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable -// condition with status True. If a VMI fails to become migratable within the -// per-attempt timeout, it is deleted and recreated. This works around an -// upstream KubeVirt race where virt-handler computes the containerdisk -// checksum before the disk socket is ready, fails, and never retries; the -// recreated VMI lands on a node that already has the containerdisk image -// cached, so the socket comes up before virt-handler's first attempt. -// See https://github.com/kubernetes-sigs/descheduler/pull/1874 for context. +// ensureVMIsLiveMigratable waits until every VMI reports the LiveMigratable condition with status True. func ensureVMIsLiveMigratable(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface, namespace string) { t.Helper() - const ( - maxAttempts = 3 - perAttemptWait = 120 * time.Second - deleteWait = 60 * time.Second - ) - - isLiveMigratable := func(vmi *kvcorev1.VirtualMachineInstance) bool { - for _, c := range vmi.Status.Conditions { - if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue { - return true - } + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 180*time.Second, true, func(ctx context.Context) (bool, error) { + vmiList, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).List(ctx, metav1.ListOptions{}) + if err != nil || len(vmiList.Items) != vmiCount { + return false, nil } - return false - } - - for i := 1; i <= vmiCount; i++ { - name := fmt.Sprintf("kubevirtvmi-%v", i) - var lastVMI *kvcorev1.VirtualMachineInstance - for attempt := 1; attempt <= maxAttempts; attempt++ { - err := wait.PollUntilContextTimeout(ctx, 5*time.Second, perAttemptWait, true, func(ctx context.Context) (bool, error) { - vmi, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - klog.Infof("Unable to get vmi %v: %v", name, err) - return false, nil - } - lastVMI = vmi - return isLiveMigratable(vmi), nil - }) - if err == nil { - klog.Infof("vmi %v is LiveMigratable (attempt %d/%d)", name, attempt, maxAttempts) - break - } - if attempt == maxAttempts { - if lastVMI != nil { - klog.Infof("Final vmi %v status: phase=%v, conditions=%#v", name, lastVMI.Status.Phase, lastVMI.Status.Conditions) + for _, vmi := range vmiList.Items { + migratable := false + for _, c := range vmi.Status.Conditions { + if c.Type == kvcorev1.VirtualMachineInstanceIsMigratable && c.Status == corev1.ConditionTrue { + migratable = true + break } - t.Fatalf("vmi %v never became LiveMigratable after %d attempts", name, maxAttempts) - } - klog.Warningf("vmi %v not LiveMigratable after %v, recreating (attempt %d/%d) to work around virt-handler containerdisk-socket race", name, perAttemptWait, attempt, maxAttempts) - if err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - t.Fatalf("Unable to delete vmi %v for retry: %v", name, err) - } - if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, deleteWait, true, func(ctx context.Context) (bool, error) { - _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Get(ctx, name, metav1.GetOptions{}) - return apierrors.IsNotFound(err), nil - }); err != nil { - t.Fatalf("Timed out waiting for vmi %v to be deleted: %v", name, err) } - if _, err := kvClient.KubevirtV1().VirtualMachineInstances(namespace).Create(ctx, virtualMachineInstance(i, namespace), metav1.CreateOptions{}); err != nil { - t.Fatalf("Unable to recreate vmi %v: %v", name, err) + if !migratable { + return false, nil } } + return true, nil + }) + if err != nil { + t.Fatalf("VMIs never became LiveMigratable: %v", err) } + klog.Infof("All VMIs are LiveMigratable") } func waitForKubevirtReady(t *testing.T, ctx context.Context, kvClient kubevirtclient.Interface) { - obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{}) - if err != nil { - t.Fatalf("Unable to get kubevirt/kubevirt: %v", err) - } - available := false - for _, condition := range obj.Status.Conditions { - if condition.Type == kvcorev1.KubeVirtConditionAvailable { - if condition.Status == corev1.ConditionTrue { - available = true + t.Helper() + err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 180*time.Second, true, func(ctx context.Context) (bool, error) { + obj, err := kvClient.KubevirtV1().KubeVirts("kubevirt").Get(ctx, "kubevirt", metav1.GetOptions{}) + if err != nil { + klog.Infof("Unable to get kubevirt/kubevirt: %v", err) + return false, nil + } + for _, condition := range obj.Status.Conditions { + if condition.Type == kvcorev1.KubeVirtConditionAvailable && condition.Status == corev1.ConditionTrue { + return true, nil } } - } - if !available { - t.Fatalf("Kubevirt is not available") + return false, nil + }) + if err != nil { + t.Fatalf("Kubevirt is not available: %v", err) } klog.Infof("Kubevirt is available") } @@ -415,7 +380,13 @@ func createAndWaitForDeschedulerRunning(t *testing.T, ctx context.Context, kubeC klog.Infof("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) _, err := kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) + if apierrors.IsAlreadyExists(err) { + _ = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) + _, err = kubeClient.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) + } + if err != nil { + t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) + } } klog.Infof("Waiting for the descheduler pod running") diff --git a/test/e2e/e2e_failedpods_test.go b/test/e2e/e2e_failedpods_test.go index bf82622df..c3e06a40b 100644 --- a/test/e2e/e2e_failedpods_test.go +++ b/test/e2e/e2e_failedpods_test.go @@ -9,7 +9,6 @@ import ( batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -149,31 +148,7 @@ func TestFailedPods(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, removeFailedPodsPolicy(tc.removeFailedPodsArgs, evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - t.Cleanup(func() { - if deschedulerPodName != "" { - printPodLogs(context.Background(), t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }) - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemoveDuplicates strategy var meetsExpectations bool diff --git a/test/e2e/e2e_lownodeutilization_test.go b/test/e2e/e2e_lownodeutilization_test.go index 225143ee7..1a79b7c03 100644 --- a/test/e2e/e2e_lownodeutilization_test.go +++ b/test/e2e/e2e_lownodeutilization_test.go @@ -25,7 +25,6 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -106,7 +105,10 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { if _, err := clientSet.CoreV1().Namespaces().Create(ctx, testNamespace, metav1.CreateOptions{}); err != nil { t.Fatalf("Unable to create ns %v: %v", testNamespace.Name, err) } - defer clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) + + t.Cleanup(func() { + clientSet.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{}) + }) t.Log("Creating duplicates pods") testLabel := map[string]string{"app": "test-lownodeutilization-kubernetes-metrics", "name": "test-lownodeutilization-kubernetes-metrics"} @@ -165,11 +167,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { expectedEvictedPodCount: 2, lowNodeUtilizationArgs: &nodeutilization.LowNodeUtilizationArgs{ Thresholds: api.ResourceThresholds{ - v1.ResourceCPU: 10, + v1.ResourceCPU: 8, v1.ResourcePods: 30, }, TargetThresholds: api.ResourceThresholds{ - v1.ResourceCPU: 20, + v1.ResourceCPU: 15, v1.ResourcePods: 50, }, MetricsUtilization: &nodeutilization.MetricsUtilization{ @@ -195,10 +197,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { } return } - defer func() { + + t.Cleanup(func() { clientSet.AppsV1().Deployments(deploymentObj.Namespace).Delete(ctx, deploymentObj.Name, metav1.DeleteOptions{}) waitForPodsToDisappear(ctx, t, clientSet, deploymentObj.Labels, deploymentObj.Namespace) - }() + }) waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace) // wait until workerNodes[0].Name has the right actual cpu utilization and all the testing pods are running // and producing ~4 cores in total @@ -239,31 +242,7 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, lowNodeUtilizationPolicy(tc.lowNodeUtilizationArgs, tc.evictorArgs, tc.metricsCollectorEnabled)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - t.Cleanup(func() { - if deschedulerPodName != "" { - printPodLogs(context.Background(), t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }) - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run LowNodeUtilization plugin var meetsExpectations bool diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0cf6b8245..2f161fd08 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -153,6 +153,43 @@ func createConfigMapWithCleanup(t *testing.T, ctx context.Context, kubeClient cl return cm } +// createDeschedulerDeploymentWithCleanup creates (or recreates if already existing) a descheduler deployment and registers a t.Cleanup callback. +func createDeschedulerDeploymentWithCleanup(t *testing.T, ctx context.Context, kubeClient clientset.Interface, deployment *appsv1.Deployment) string { + t.Helper() + t.Logf("Creating descheduler deployment %v", deployment.Name) + _, err := kubeClient.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + _ = kubeClient.AppsV1().Deployments(deployment.Namespace).Delete(ctx, deployment.Name, metav1.DeleteOptions{}) + _, err = kubeClient.AppsV1().Deployments(deployment.Namespace).Create(ctx, deployment, metav1.CreateOptions{}) + } + if err != nil { + t.Fatalf("Error creating %q deployment: %v", deployment.Name, err) + } + } + + deschedulerPodName := "" + t.Cleanup(func() { + if deschedulerPodName != "" { + printPodLogs(context.Background(), t, kubeClient, deschedulerPodName) + } + + t.Logf("Deleting %q deployment...", deployment.Name) + if err := kubeClient.AppsV1().Deployments(deployment.Namespace).Delete(context.Background(), deployment.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Logf("Unable to delete %q deployment: %v", deployment.Name, err) + } + + waitForPodsToDisappear(context.Background(), t, kubeClient, deployment.Labels, deployment.Namespace) + }) + + t.Logf("Waiting for the descheduler pod running") + deschedulerPods := waitForPodsRunning(ctx, t, kubeClient, deployment.Labels, 1, deployment.Namespace) + if len(deschedulerPods) != 0 { + deschedulerPodName = deschedulerPods[0].Name + } + return deschedulerPodName +} + func deschedulerDeployment(testName string) *appsv1.Deployment { deploymentObject := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/e2e_toomanyrestarts_test.go b/test/e2e/e2e_toomanyrestarts_test.go index d4fee2b43..db264314f 100644 --- a/test/e2e/e2e_toomanyrestarts_test.go +++ b/test/e2e/e2e_toomanyrestarts_test.go @@ -156,31 +156,7 @@ func TestTooManyRestarts(t *testing.T) { createPolicyConfigMap(t, ctx, clientSet, tc.policy) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - defer func() { - if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemovePodsHavingTooManyRestarts strategy if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (bool, error) { currentRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...) diff --git a/test/e2e/e2e_topologyspreadconstraint_test.go b/test/e2e/e2e_topologyspreadconstraint_test.go index 3a10c2942..129cc865b 100644 --- a/test/e2e/e2e_topologyspreadconstraint_test.go +++ b/test/e2e/e2e_topologyspreadconstraint_test.go @@ -221,31 +221,7 @@ func TestTopologySpreadConstraint(t *testing.T) { } createPolicyConfigMap(t, ctx, clientSet, topologySpreadConstraintPolicy(constraintArgs, evictorArgs)) deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name) - t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name) - _, err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Create(ctx, deschedulerDeploymentObj, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Error creating %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - - deschedulerPodName := "" - defer func() { - if deschedulerPodName != "" { - printPodLogs(ctx, t, clientSet, deschedulerPodName) - } - - t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name) - err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err) - } - waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace) - }() - - t.Logf("Waiting for the descheduler pod running") - deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace) - if len(deschedulerPods) != 0 { - deschedulerPodName = deschedulerPods[0].Name - } + createDeschedulerDeploymentWithCleanup(t, ctx, clientSet, deschedulerDeploymentObj) // Run RemovePodsHavingTooManyRestarts strategy var meetsEvictedExpectations bool