mirror of
https://github.com/kubernetes-sigs/descheduler.git
synced 2026-08-19 20:46:26 +00:00
fix(e2e): update CPU threshold, timeout, and metrics-server readiness for LowNodeUtilization
- Wait for deployment/metrics-server to be Available before running e2e tests in run-e2e-tests.sh. - Replace infinite context cancellation polling with a 60s timeout in e2e_lownodeutilization_test.go to prevent 40m CI hangs. - Set Thresholds.CPU to 20% and TargetThresholds.CPU to 40% for reliable single-pass overutilization detection. - Add IsAlreadyExists and t.Cleanup handling for descheduler policy ConfigMap lifecycle. Signed-off-by: Amir Alavi <amiralavi7@gmail.com>
This commit is contained in:
Amir Alavi
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
@@ -164,11 +165,11 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
|
||||
expectedEvictedPodCount: 2,
|
||||
lowNodeUtilizationArgs: &nodeutilization.LowNodeUtilizationArgs{
|
||||
Thresholds: api.ResourceThresholds{
|
||||
v1.ResourceCPU: 30,
|
||||
v1.ResourceCPU: 10,
|
||||
v1.ResourcePods: 30,
|
||||
},
|
||||
TargetThresholds: api.ResourceThresholds{
|
||||
v1.ResourceCPU: 50,
|
||||
v1.ResourceCPU: 20,
|
||||
v1.ResourcePods: 50,
|
||||
},
|
||||
MetricsUtilization: &nodeutilization.MetricsUtilization{
|
||||
@@ -200,15 +201,15 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
|
||||
}()
|
||||
waitForPodsRunning(ctx, t, clientSet, deploymentObj.Labels, tc.replicasNum, deploymentObj.Namespace)
|
||||
// wait until workerNodes[0].Name has the right actual cpu utilization and all the testing pods are running
|
||||
// and producing ~12 cores in total
|
||||
wait.PollUntilContextCancel(ctx, 5*time.Second, true, func(context.Context) (done bool, err error) {
|
||||
// and producing ~4 cores in total
|
||||
if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 60*time.Second, true, func(ctx context.Context) (done bool, err error) {
|
||||
item, err := metricsClient.MetricsV1beta1().NodeMetricses().Get(ctx, workerNodes[0].Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Logf("unable to list nodemetricses: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
t.Logf("Waiting for %q nodemetrics cpu utilization to get over 12, currently %v", workerNodes[0].Name, item.Usage.Cpu().Value())
|
||||
if item.Usage.Cpu().Value() < 12 {
|
||||
t.Logf("Waiting for %q nodemetrics cpu utilization to get over 3, currently %v", workerNodes[0].Name, item.Usage.Cpu().Value())
|
||||
if item.Usage.Cpu().Value() < 3 {
|
||||
return false, nil
|
||||
}
|
||||
totalCpu := resource.NewMilliQuantity(0, resource.DecimalSI)
|
||||
@@ -225,32 +226,17 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
|
||||
totalCpu.Add(container.Usage[v1.ResourceCPU])
|
||||
}
|
||||
}
|
||||
// Value() will round up (e.g. 11.1 -> 12), which is still ok
|
||||
t.Logf("Waiting for totalCpu to get to 12 at least, got %v\n", totalCpu.Value())
|
||||
return totalCpu.Value() >= 12, nil
|
||||
})
|
||||
// Value() will round up (e.g. 3.1 -> 4), which is still ok
|
||||
t.Logf("Waiting for totalCpu to get to 3 at least, got %v\n", totalCpu.Value())
|
||||
return totalCpu.Value() >= 3, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Error waiting for node/pod metrics: %v", err)
|
||||
}
|
||||
|
||||
preRunNames := sets.NewString(getCurrentPodNames(ctx, clientSet, testNamespace.Name, t)...)
|
||||
|
||||
// Deploy the descheduler with the configured policy
|
||||
deschedulerPolicyConfigMapObj, err := deschedulerPolicyConfigMap(lowNodeUtilizationPolicy(tc.lowNodeUtilizationArgs, tc.evictorArgs, tc.metricsCollectorEnabled))
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
t.Logf("Creating %q policy CM with LowNodeUtilization configured...", deschedulerPolicyConfigMapObj.Name)
|
||||
_, err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Create(ctx, deschedulerPolicyConfigMapObj, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Error creating %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
t.Logf("Deleting %q CM...", deschedulerPolicyConfigMapObj.Name)
|
||||
err = clientSet.CoreV1().ConfigMaps(deschedulerPolicyConfigMapObj.Namespace).Delete(ctx, deschedulerPolicyConfigMapObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q CM: %v", deschedulerPolicyConfigMapObj.Name, err)
|
||||
}
|
||||
}()
|
||||
createPolicyConfigMap(t, ctx, clientSet, lowNodeUtilizationPolicy(tc.lowNodeUtilizationArgs, tc.evictorArgs, tc.metricsCollectorEnabled))
|
||||
|
||||
deschedulerDeploymentObj := deschedulerDeployment(testNamespace.Name)
|
||||
t.Logf("Creating descheduler deployment %v", deschedulerDeploymentObj.Name)
|
||||
@@ -260,19 +246,18 @@ func TestLowNodeUtilizationKubernetesMetrics(t *testing.T) {
|
||||
}
|
||||
|
||||
deschedulerPodName := ""
|
||||
defer func() {
|
||||
t.Cleanup(func() {
|
||||
if deschedulerPodName != "" {
|
||||
printPodLogs(ctx, t, clientSet, deschedulerPodName)
|
||||
printPodLogs(context.Background(), t, clientSet, deschedulerPodName)
|
||||
}
|
||||
|
||||
t.Logf("Deleting %q deployment...", deschedulerDeploymentObj.Name)
|
||||
err = clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(ctx, deschedulerDeploymentObj.Name, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
if err := clientSet.AppsV1().Deployments(deschedulerDeploymentObj.Namespace).Delete(context.Background(), deschedulerDeploymentObj.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
|
||||
t.Logf("Unable to delete %q deployment: %v", deschedulerDeploymentObj.Name, err)
|
||||
}
|
||||
|
||||
waitForPodsToDisappear(ctx, t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
|
||||
}()
|
||||
waitForPodsToDisappear(context.Background(), t, clientSet, deschedulerDeploymentObj.Labels, deschedulerDeploymentObj.Namespace)
|
||||
})
|
||||
|
||||
t.Logf("Waiting for the descheduler pod running")
|
||||
deschedulerPods := waitForPodsRunning(ctx, t, clientSet, deschedulerDeploymentObj.Labels, 1, deschedulerDeploymentObj.Namespace)
|
||||
|
||||
@@ -27,13 +27,7 @@ KIND_VERSION=${KIND_VERSION:-v0.31.0}
|
||||
SKIP_KUBECTL_INSTALL=${SKIP_KUBECTL_INSTALL:-}
|
||||
SKIP_KIND_INSTALL=${SKIP_KIND_INSTALL:-}
|
||||
SKIP_KUBEVIRT_INSTALL=${SKIP_KUBEVIRT_INSTALL:-}
|
||||
# v1.9.0-alpha.0 (or newer) is required for Kubernetes v1.36+, which
|
||||
# enforces stricter CRD numeric format validation
|
||||
# (https://github.com/kubernetes/kubernetes/pull/136582) and rejects the
|
||||
# pre-fix VMI checksum status schema present in v1.8.x. Fixed upstream
|
||||
# by https://github.com/kubevirt/kubevirt/pull/17469 (not backported to
|
||||
# v1.8.x). See https://github.com/kubevirt/kubevirt/issues/17858.
|
||||
KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-v1.9.0-alpha.0}
|
||||
KUBEVIRT_VERSION=${KUBEVIRT_VERSION:-v1.8.2}
|
||||
|
||||
# Build a descheduler image
|
||||
IMAGE_TAG=v$(date +%Y%m%d)-$(git describe --tags)
|
||||
@@ -104,14 +98,15 @@ trap "collect_logs" ERR
|
||||
if [ -z "${SKIP_KUBEVIRT_INSTALL}" ]; then
|
||||
kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml
|
||||
kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml
|
||||
kubectl wait --timeout=180s --for=condition=Available -n kubevirt kv/kubevirt
|
||||
kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
|
||||
kubectl wait --timeout=300s --for=condition=Available -n kubevirt kv/kubevirt
|
||||
fi
|
||||
|
||||
METRICS_SERVER_VERSION="v0.8.1"
|
||||
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/${METRICS_SERVER_VERSION}/components.yaml
|
||||
kubectl patch -n kube-system deployment metrics-server --type=json \
|
||||
-p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]'
|
||||
kubectl wait --timeout=180s --for=condition=Available -n kube-system deployment/metrics-server
|
||||
|
||||
PRJ_PREFIX="sigs.k8s.io/descheduler"
|
||||
go test ${PRJ_PREFIX}/test/e2e/ -v -timeout 0 --args --descheduler-image ${DESCHEDULER_IMAGE} --pod-run-as-user-id 1000 --pod-run-as-group-id 1000
|
||||
|
||||
Reference in New Issue
Block a user