Merge pull request #1866 from ingvagabund/automated-cherry-pick-of-#1862-upstream-release-1.35

Automated cherry pick of #1862: fix(descheduler): reset prometheus usage client at each

.github/workflows/manifests.yaml

@@ -8,22 +8,22 @@ jobs:
     strategy:
       matrix:
         k8s-version: ["v1.35.1"]
-        descheduler-version: ["v0.35.0"]
+        descheduler-version: ["v0.35.1"]
         descheduler-api: ["v1alpha2"]
         manifest: ["deployment"]
         kind-version: ["v0.31.0"] # keep in sync with test/run-e2e-tests.sh
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # git ls-remote https://github.com/actions/checkout.git refs/tags/v6.0.2
       - name: Create kind cluster
-        uses: helm/kind-action@v1.12.0
+        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # git ls-remote https://github.com/helm/kind-action.git refs/tags/v1.14.0
         with:
           node_image: kindest/node:${{ matrix.k8s-version }}
           kubectl_version: ${{ matrix.k8s-version }}
           config: test/kind-config.yaml
           version: ${{ matrix.kind-version }}
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # git ls-remote https://github.com/actions/setup-go.git refs/tags/v6.4.0
         with:
           go-version-file: go.mod
           cache: true

charts/descheduler/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: descheduler
-version: 0.35.0
-appVersion: 0.35.0
+version: 0.35.1
+appVersion: 0.35.1
 description: Descheduler for Kubernetes is used to rebalance clusters by evicting pods that can potentially be scheduled on better nodes. In the current implementation, descheduler does not schedule replacement of evicted pods but relies on the default scheduler for that.
 keywords:
 - kubernetes

docs/user-guide.md

@@ -4,6 +4,7 @@ Starting with descheduler release v0.10.0 container images are available in the
 
 Descheduler Version | Container Image                                 | Architectures           |
 ------------------- |-------------------------------------------------|-------------------------|
+v0.35.1             | registry.k8s.io/descheduler/descheduler:v0.35.1 | AMD64<br>ARM64<br>ARMv7 |
 v0.35.0             | registry.k8s.io/descheduler/descheduler:v0.35.0 | AMD64<br>ARM64<br>ARMv7 |
 v0.34.0             | registry.k8s.io/descheduler/descheduler:v0.34.0 | AMD64<br>ARM64<br>ARMv7 |
 v0.33.0             | registry.k8s.io/descheduler/descheduler:v0.33.0 | AMD64<br>ARM64<br>ARMv7 |

kubernetes/cronjob/cronjob.yaml

@@ -16,7 +16,7 @@ spec:
           priorityClassName: system-cluster-critical
           containers:
           - name: descheduler
-            image: registry.k8s.io/descheduler/descheduler:v0.35.0
+            image: registry.k8s.io/descheduler/descheduler:v0.35.1
             volumeMounts:
             - mountPath: /policy-dir
               name: policy-volume

kubernetes/deployment/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: descheduler-sa
       containers:
         - name: descheduler
-          image: registry.k8s.io/descheduler/descheduler:v0.35.0
+          image: registry.k8s.io/descheduler/descheduler:v0.35.1
           imagePullPolicy: IfNotPresent
           command:
             - "/bin/descheduler"

kubernetes/job/job.yaml

@@ -14,7 +14,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: descheduler
-          image: registry.k8s.io/descheduler/descheduler:v0.35.0
+          image: registry.k8s.io/descheduler/descheduler:v0.35.1
           volumeMounts:
           - mountPath: /policy-dir
             name: policy-volume

pkg/framework/plugins/nodeutilization/lownodeutilization.go

@@ -103,18 +103,13 @@ func NewLowNodeUtilization(
 	}
 
 	// this plugins supports different ways of collecting usage data. each
-	// different way provides its own "usageClient". here we make sure we
-	// have the correct one or an error is triggered. XXX MetricsServer is
-	// deprecated, removed once dropped.
+	// different way provides its own "usageClient". the metrics-based client
+	// is reset at every extension point so we always get the latest prometheus
+	// client (whose SA token can be rotated after plugin creation).
+	// XXX MetricsServer is deprecated, removed once dropped.
 	var usageClient usageClient = newRequestedUsageClient(
 		extendedResourceNames, handle.GetPodsAssignedToNodeFunc(),
 	)
-	if metrics != nil {
-		usageClient, err = usageClientForMetrics(args, handle, extendedResourceNames)
-		if err != nil {
-			return nil, err
-		}
-	}
 
 	return &LowNodeUtilization{
 		logger:                logger,
@@ -134,12 +129,33 @@ func (l *LowNodeUtilization) Name() string {
 	return LowNodeUtilizationPluginName
 }
 
+// resetUsageClient refreshes the usageClient field from the current handle
+// state. It must be called at the start of every extension point so that a
+// rotated prometheus SA token is picked up without restarting the process.
+func (l *LowNodeUtilization) resetUsageClient() error {
+	if l.args.MetricsUtilization == nil {
+		return nil
+	}
+	client, err := usageClientForMetrics(l.args, l.handle, l.extendedResourceNames)
+	if err != nil {
+		return err
+	}
+	l.usageClient = client
+	return nil
+}
+
 // Balance holds the main logic of the plugin. It evicts pods from over
 // utilized nodes to under utilized nodes. The goal here is to evenly
 // distribute pods across nodes.
 func (l *LowNodeUtilization) Balance(ctx context.Context, nodes []*v1.Node) *frameworktypes.Status {
 	logger := klog.FromContext(klog.NewContext(ctx, l.logger)).WithValues("ExtensionPoint", frameworktypes.BalanceExtensionPoint)
 
+	if err := l.resetUsageClient(); err != nil {
+		return &frameworktypes.Status{
+			Err: fmt.Errorf("error initializing usage client: %v", err),
+		}
+	}
+
 	if err := l.usageClient.sync(ctx, nodes); err != nil {
 		return &frameworktypes.Status{
 			Err: fmt.Errorf("error getting node usage: %v", err),

pkg/framework/plugins/nodeutilization/lownodeutilization_test.go

@@ -1661,3 +1661,83 @@ func TestLowNodeUtilizationWithPrometheusMetrics(t *testing.T) {
 		t.Run(tc.name, testFnc(false, tc.expectedPodsEvicted))
 	}
 }
+
+// TestLowNodeUtilizationPrometheusClientNilAtCreation ensures the plugin
+// tolerates a nil prometheus client at creation time (e.g. in-cluster SA token
+// not yet reconciled) and correctly picks up the client exposed by the handle
+// at Balance call time.
+func TestLowNodeUtilizationPrometheusClientNilAtCreation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	n1NodeName := "n1"
+	n2NodeName := "n2"
+	n3NodeName := "n3"
+
+	nodes := []*v1.Node{
+		test.BuildTestNode(n1NodeName, 4000, 3000, 9, nil),
+		test.BuildTestNode(n2NodeName, 4000, 3000, 10, nil),
+		test.BuildTestNode(n3NodeName, 4000, 3000, 10, nil),
+	}
+	pods := []*v1.Pod{
+		test.BuildTestPod("p1", 400, 0, n1NodeName, test.SetRSOwnerRef),
+		test.BuildTestPod("p2", 400, 0, n1NodeName, test.SetRSOwnerRef),
+		test.BuildTestPod("p3", 400, 0, n1NodeName, test.SetRSOwnerRef),
+		test.BuildTestPod("p4", 400, 0, n1NodeName, test.SetRSOwnerRef),
+		test.BuildTestPod("p5", 400, 0, n1NodeName, test.SetRSOwnerRef),
+		test.BuildTestPod("p6", 400, 0, n2NodeName, test.SetRSOwnerRef),
+	}
+
+	var objs []runtime.Object
+	for _, node := range nodes {
+		objs = append(objs, node)
+	}
+	for _, pod := range pods {
+		objs = append(objs, pod)
+	}
+	fakeClient := fake.NewSimpleClientset(objs...)
+
+	handle, podEvictor, err := frameworktesting.InitFrameworkHandle(
+		ctx, fakeClient, nil, defaultevictor.DefaultEvictorArgs{NodeFit: true}, nil,
+	)
+	if err != nil {
+		t.Fatalf("Unable to initialize a framework handle: %v", err)
+	}
+
+	// PrometheusClientImpl is intentionally nil here — the in-cluster SA token
+	// has not been reconciled yet at plugin-creation time.
+	args := &LowNodeUtilizationArgs{
+		Thresholds:       api.ResourceThresholds{MetricResource: 30},
+		TargetThresholds: api.ResourceThresholds{MetricResource: 50},
+		MetricsUtilization: &MetricsUtilization{
+			Source: api.PrometheusMetrics,
+			Prometheus: &Prometheus{
+				Query: "instance:node_cpu:rate:sum",
+			},
+		},
+	}
+
+	plugin, err := NewLowNodeUtilization(ctx, args, handle)
+	if err != nil {
+		t.Fatalf("plugin creation must succeed even when prometheus client is nil: %v", err)
+	}
+
+	// Simulate the SA token becoming available before the first descheduling cycle.
+	handle.PrometheusClientImpl = &fakePromClient{
+		result: model.Vector{
+			sample("instance:node_cpu:rate:sum", n1NodeName, 0.57), // over target
+			sample("instance:node_cpu:rate:sum", n2NodeName, 0.42), // over target
+			sample("instance:node_cpu:rate:sum", n3NodeName, 0.20), // under threshold
+		},
+		dataType: model.ValVector,
+	}
+
+	status := plugin.(frameworktypes.BalancePlugin).Balance(ctx, nodes)
+	if status != nil {
+		t.Fatalf("Balance failed: %v", status.Err)
+	}
+
+	if podEvictor.TotalEvicted() == 0 {
+		t.Error("expected at least one pod to be evicted")
+	}
+}