Merge pull request #1821 from ingvagabund/prom-client-controllers

refactor(promClientController): split it into two prom client controllers
2026-08-23 22:46:35 +00:00 · 2026-02-04 22:48:32 +05:30
parent 5fb70c12c7 d262c7af44
commit fc863ff58d
2 changed files with 390 additions and 190 deletions
@@ -22,6 +22,7 @@ import (
 	"math"
 	"net/http"
 	"strconv"
+	"sync"
 	"time"

 	promapi "github.com/prometheus/client_golang/api"
@@ -78,17 +79,41 @@ type profileRunner struct {
 	descheduleEPs, balanceEPs eprunner
 }

+// inClusterPromClientController manages prometheus client using in-cluster SA token
+type inClusterPromClientController struct {
+	mu                                sync.RWMutex
+	promClient                        promapi.Client
+	previousPrometheusClientTransport *http.Transport
+	currentPrometheusAuthToken        string
+	prometheusConfig                  *api.Prometheus
+	createPrometheusClient            createPrometheusClientFunc
+	inClusterConfig                   inClusterConfigFunc
+}
+
+// secretBasedPromClientController manages prometheus client using Kubernetes secret
+type secretBasedPromClientController struct {
+	mu                                sync.RWMutex
+	promClient                        promapi.Client
+	previousPrometheusClientTransport *http.Transport
+	queue                             workqueue.RateLimitingInterface
+	currentPrometheusAuthToken        string
+	namespacedSecretsLister           corev1listers.SecretNamespaceLister
+	prometheusConfig                  *api.Prometheus
+	createPrometheusClient            createPrometheusClientFunc
+}
+
 type descheduler struct {
-	rs                    *options.DeschedulerServer
-	client                clientset.Interface
-	kubeClientSandbox     *kubeClientSandbox
-	getPodsAssignedToNode podutil.GetPodsAssignedToNodeFunc
-	sharedInformerFactory informers.SharedInformerFactory
-	deschedulerPolicy     *api.DeschedulerPolicy
-	eventRecorder         events.EventRecorder
-	podEvictor            *evictions.PodEvictor
-	metricsCollector      *metricscollector.MetricsCollector
-	promClientCtrl        *promClientController
+	rs                        *options.DeschedulerServer
+	client                    clientset.Interface
+	kubeClientSandbox         *kubeClientSandbox
+	getPodsAssignedToNode     podutil.GetPodsAssignedToNodeFunc
+	sharedInformerFactory     informers.SharedInformerFactory
+	deschedulerPolicy         *api.DeschedulerPolicy
+	eventRecorder             events.EventRecorder
+	podEvictor                *evictions.PodEvictor
+	metricsCollector          *metricscollector.MetricsCollector
+	inClusterPromClientCtrl   *inClusterPromClientController
+	secretBasedPromClientCtrl *secretBasedPromClientController
 }

 type (
@@ -96,28 +121,50 @@ type (
 	inClusterConfigFunc        func() (*rest.Config, error)
 )

-type promClientController struct {
-	promClient                        promapi.Client
-	previousPrometheusClientTransport *http.Transport
-	queue                             workqueue.RateLimitingInterface
-	currentPrometheusAuthToken        string
-	namespacedSecretsLister           corev1listers.SecretNamespaceLister
-	metricsProviders                  map[api.MetricsSource]*api.MetricsProvider
-	createPrometheusClient            createPrometheusClientFunc
-	inClusterConfig                   inClusterConfigFunc
-}
-
-func newPromClientController(prometheusClient promapi.Client, metricsProviders map[api.MetricsSource]*api.MetricsProvider) *promClientController {
-	return &promClientController{
+func newInClusterPromClientController(prometheusClient promapi.Client, prometheusConfig *api.Prometheus) *inClusterPromClientController {
+	return &inClusterPromClientController{
 		promClient:             prometheusClient,
-		queue:                  workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "descheduler"}),
-		metricsProviders:       metricsProviders,
+		prometheusConfig:       prometheusConfig,
 		createPrometheusClient: client.CreatePrometheusClient,
 		inClusterConfig:        rest.InClusterConfig,
 	}
 }

-func (d *promClientController) prometheusClient() promapi.Client {
+func newSecretBasedPromClientController(prometheusClient promapi.Client, prometheusConfig *api.Prometheus, namespacedSharedInformerFactory informers.SharedInformerFactory) (*secretBasedPromClientController, error) {
+	if prometheusConfig == nil || prometheusConfig.AuthToken == nil || prometheusConfig.AuthToken.SecretReference == nil {
+		return nil, fmt.Errorf("prometheus metrics source configuration is missing authentication token secret")
+	}
+	authTokenSecret := prometheusConfig.AuthToken.SecretReference
+	if authTokenSecret.Name == "" || authTokenSecret.Namespace == "" {
+		return nil, fmt.Errorf("prometheus metrics source configuration is missing authentication token secret")
+	}
+
+	if namespacedSharedInformerFactory == nil {
+		return nil, fmt.Errorf("namespacedSharedInformerFactory not configured")
+	}
+
+	ctrl := &secretBasedPromClientController{
+		promClient:             prometheusClient,
+		queue:                  workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{Name: "descheduler"}),
+		prometheusConfig:       prometheusConfig,
+		createPrometheusClient: client.CreatePrometheusClient,
+	}
+
+	namespacedSharedInformerFactory.Core().V1().Secrets().Informer().AddEventHandler(ctrl.eventHandler())
+	ctrl.namespacedSecretsLister = namespacedSharedInformerFactory.Core().V1().Secrets().Lister().Secrets(authTokenSecret.Namespace)
+
+	return ctrl, nil
+}
+
+func (d *inClusterPromClientController) prometheusClient() promapi.Client {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
+	return d.promClient
+}
+
+func (d *secretBasedPromClientController) prometheusClient() promapi.Client {
+	d.mu.RLock()
+	defer d.mu.RUnlock()
 	return d.promClient
 }

@@ -140,32 +187,25 @@ func addNodeSelectorIndexer(sharedInformerFactory informers.SharedInformerFactor
 func metricsProviderListToMap(providersList []api.MetricsProvider) map[api.MetricsSource]*api.MetricsProvider {
 	providersMap := make(map[api.MetricsSource]*api.MetricsProvider)
 	for _, provider := range providersList {
+		provider := provider
 		providersMap[provider.Source] = &provider
 	}
 	return providersMap
 }

-// setupPrometheusProvider sets up the prometheus provider on the descheduler if configured
-func setupPrometheusProvider(d *descheduler, namespacedSharedInformerFactory informers.SharedInformerFactory) error {
-	if d.promClientCtrl == nil {
+func getPrometheusConfig(providersList []api.MetricsProvider) *api.Prometheus {
+	prometheusProvider := metricsProviderListToMap(providersList)[api.PrometheusMetrics]
+	if prometheusProvider == nil {
 		return nil
 	}
-	prometheusProvider := d.promClientCtrl.metricsProviders[api.PrometheusMetrics]
-	if prometheusProvider != nil && prometheusProvider.Prometheus != nil && prometheusProvider.Prometheus.AuthToken != nil {
-		authTokenSecret := prometheusProvider.Prometheus.AuthToken.SecretReference
-		if authTokenSecret == nil || authTokenSecret.Namespace == "" {
-			return fmt.Errorf("prometheus metrics source configuration is missing authentication token secret")
-		}
-		if namespacedSharedInformerFactory == nil {
-			return fmt.Errorf("namespacedSharedInformerFactory not configured")
-		}
-		namespacedSharedInformerFactory.Core().V1().Secrets().Informer().AddEventHandler(d.promClientCtrl.eventHandler())
-		d.promClientCtrl.namespacedSecretsLister = namespacedSharedInformerFactory.Core().V1().Secrets().Lister().Secrets(authTokenSecret.Namespace)
-	}
-	return nil
+	return prometheusProvider.Prometheus
 }

-func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string, eventRecorder events.EventRecorder, client clientset.Interface, sharedInformerFactory informers.SharedInformerFactory, kubeClientSandbox *kubeClientSandbox) (*descheduler, error) {
+func configureSecretPromClientReconciler(prometheusConfig *api.Prometheus) bool {
+	return prometheusConfig.URL != "" && prometheusConfig.AuthToken != nil
+}
+
+func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string, eventRecorder events.EventRecorder, client clientset.Interface, sharedInformerFactory, namespacedSharedInformerFactory informers.SharedInformerFactory, kubeClientSandbox *kubeClientSandbox) (*descheduler, error) {
 	podInformer := sharedInformerFactory.Core().V1().Pods().Informer()
 	// Temporarily register the PVC because it is used by the DefaultEvictor plugin during
 	// the descheduling cycle, where informer registration is ignored.
@@ -205,7 +245,21 @@ func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedu
 		deschedulerPolicy:     deschedulerPolicy,
 		eventRecorder:         eventRecorder,
 		podEvictor:            podEvictor,
-		promClientCtrl:        newPromClientController(rs.PrometheusClient, metricsProviderListToMap(deschedulerPolicy.MetricsProviders)),
+	}
+
+	prometheusConfig := getPrometheusConfig(deschedulerPolicy.MetricsProviders)
+	if prometheusConfig != nil && prometheusConfig.URL != "" {
+		if configureSecretPromClientReconciler(prometheusConfig) {
+			// Secret-based mode
+			ctrl, err := newSecretBasedPromClientController(rs.PrometheusClient, prometheusConfig, namespacedSharedInformerFactory)
+			if err != nil {
+				return nil, err
+			}
+			desch.secretBasedPromClientCtrl = ctrl
+		} else {
+			// In-cluster mode
+			desch.inClusterPromClientCtrl = newInClusterPromClientController(rs.PrometheusClient, prometheusConfig)
+		}
 	}

 	nodeSelector, err := nodeSelectorFromPolicy(deschedulerPolicy)
@@ -224,14 +278,18 @@ func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedu
 	return desch, nil
 }

-func (d *promClientController) reconcileInClusterSAToken() error {
+func (d *inClusterPromClientController) reconcileInClusterSAToken() error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
 	// Read the sa token and assume it has the sufficient permissions to authenticate
 	cfg, err := d.inClusterConfig()
 	if err == nil {
 		if d.currentPrometheusAuthToken != cfg.BearerToken {
 			klog.V(2).Infof("Creating Prometheus client (with SA token)")
-			prometheusClient, transport, err := d.createPrometheusClient(d.metricsProviders[api.PrometheusMetrics].Prometheus.URL, cfg.BearerToken)
+			prometheusClient, transport, err := d.createPrometheusClient(d.prometheusConfig.URL, cfg.BearerToken)
 			if err != nil {
+				d.clearConnection()
 				return fmt.Errorf("unable to create a prometheus client: %v", err)
 			}
 			d.promClient = prometheusClient
@@ -249,7 +307,24 @@ func (d *promClientController) reconcileInClusterSAToken() error {
 	return fmt.Errorf("unexpected error when reading in cluster config: %v", err)
 }

-func (d *promClientController) runAuthenticationSecretReconciler(ctx context.Context) {
+func clearPromClientConnection(currentPrometheusAuthToken *string, previousPrometheusClientTransport **http.Transport, promClient *promapi.Client) {
+	*currentPrometheusAuthToken = ""
+	if *previousPrometheusClientTransport != nil {
+		(*previousPrometheusClientTransport).CloseIdleConnections()
+	}
+	*previousPrometheusClientTransport = nil
+	*promClient = nil
+}
+
+func (d *inClusterPromClientController) clearConnection() {
+	clearPromClientConnection(&d.currentPrometheusAuthToken, &d.previousPrometheusClientTransport, &d.promClient)
+}
+
+func (d *secretBasedPromClientController) clearConnection() {
+	clearPromClientConnection(&d.currentPrometheusAuthToken, &d.previousPrometheusClientTransport, &d.promClient)
+}
+
+func (d *secretBasedPromClientController) runAuthenticationSecretReconciler(ctx context.Context) {
 	defer utilruntime.HandleCrash()
 	defer d.queue.ShutDown()

@@ -261,12 +336,12 @@ func (d *promClientController) runAuthenticationSecretReconciler(ctx context.Con
 	<-ctx.Done()
 }

-func (d *promClientController) runAuthenticationSecretReconcilerWorker(ctx context.Context) {
+func (d *secretBasedPromClientController) runAuthenticationSecretReconcilerWorker(ctx context.Context) {
 	for d.processNextWorkItem(ctx) {
 	}
 }

-func (d *promClientController) processNextWorkItem(ctx context.Context) bool {
+func (d *secretBasedPromClientController) processNextWorkItem(ctx context.Context) bool {
 	dsKey, quit := d.queue.Get()
 	if quit {
 		return false
@@ -285,28 +360,24 @@ func (d *promClientController) processNextWorkItem(ctx context.Context) bool {
 	return true
 }

-func (d *promClientController) sync() error {
-	prometheusConfig := d.metricsProviders[api.PrometheusMetrics].Prometheus
-	if prometheusConfig == nil || prometheusConfig.AuthToken == nil || prometheusConfig.AuthToken.SecretReference == nil {
-		return fmt.Errorf("prometheus metrics source configuration is missing authentication token secret")
-	}
+func (d *secretBasedPromClientController) sync() error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	prometheusConfig := d.prometheusConfig
 	ns := prometheusConfig.AuthToken.SecretReference.Namespace
 	name := prometheusConfig.AuthToken.SecretReference.Name
 	secretObj, err := d.namespacedSecretsLister.Get(name)
 	if err != nil {
 		// clear the token if the secret is not found
 		if apierrors.IsNotFound(err) {
-			d.currentPrometheusAuthToken = ""
-			if d.previousPrometheusClientTransport != nil {
-				d.previousPrometheusClientTransport.CloseIdleConnections()
-			}
-			d.previousPrometheusClientTransport = nil
-			d.promClient = nil
+			d.clearConnection()
 		}
 		return fmt.Errorf("unable to get %v/%v secret", ns, name)
 	}
 	authToken := string(secretObj.Data[prometheusAuthTokenSecretKey])
 	if authToken == "" {
+		d.clearConnection()
 		return fmt.Errorf("prometheus authentication token secret missing %q data or empty", prometheusAuthTokenSecretKey)
 	}
 	if d.currentPrometheusAuthToken == authToken {
@@ -316,6 +387,7 @@ func (d *promClientController) sync() error {
 	klog.V(2).Infof("authentication secret token updated, recreating prometheus client")
 	prometheusClient, transport, err := d.createPrometheusClient(prometheusConfig.URL, authToken)
 	if err != nil {
+		d.clearConnection()
 		return fmt.Errorf("unable to create a prometheus client: %v", err)
 	}
 	d.promClient = prometheusClient
@@ -327,7 +399,7 @@ func (d *promClientController) sync() error {
 	return nil
 }

-func (d *promClientController) eventHandler() cache.ResourceEventHandler {
+func (d *secretBasedPromClientController) eventHandler() cache.ResourceEventHandler {
 	return cache.ResourceEventHandlerFuncs{
 		AddFunc:    func(obj interface{}) { d.queue.Add(workQueueKey) },
 		UpdateFunc: func(old, new interface{}) { d.queue.Add(workQueueKey) },
@@ -397,8 +469,14 @@ func (d *descheduler) runProfiles(ctx context.Context) {
 	var profileRunners []profileRunner
 	for idx, profile := range d.deschedulerPolicy.Profiles {
 		var promClient promapi.Client
-		if d.promClientCtrl != nil {
-			promClient = d.promClientCtrl.prometheusClient()
+		if d.inClusterPromClientCtrl != nil && d.secretBasedPromClientCtrl != nil {
+			klog.Error(fmt.Errorf("At most one of inClusterPromClientCtrl and secretBasedPromClientCtrl can be set, got both"))
+			continue
+		}
+		if d.inClusterPromClientCtrl != nil {
+			promClient = d.inClusterPromClientCtrl.prometheusClient()
+		} else if d.secretBasedPromClientCtrl != nil {
+			promClient = d.secretBasedPromClientCtrl.prometheusClient()
 		}
 		currProfile, err := frameworkprofile.NewProfile(
 			ctx,
@@ -539,14 +617,6 @@ func validateVersionCompatibility(discovery discovery.DiscoveryInterface, desche
 	return nil
 }

-type tokenReconciliation int
-
-const (
-	noReconciliation tokenReconciliation = iota
-	inClusterReconciliation
-	secretReconciliation
-)
-
 type runFncType func(context.Context) error

 func bootstrapDescheduler(
@@ -554,21 +624,15 @@ func bootstrapDescheduler(
 	rs *options.DeschedulerServer,
 	deschedulerPolicy *api.DeschedulerPolicy,
 	evictionPolicyGroupVersion string,
-	metricProviderTokenReconciliation tokenReconciliation,
 	sharedInformerFactory, namespacedSharedInformerFactory informers.SharedInformerFactory,
 	eventRecorder events.EventRecorder,
 ) (*descheduler, runFncType, error) {
 	// Always create descheduler with real client/factory first to register all informers
-	descheduler, err := newDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, eventRecorder, rs.Client, sharedInformerFactory, nil)
+	descheduler, err := newDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, eventRecorder, rs.Client, sharedInformerFactory, namespacedSharedInformerFactory, nil)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to create new descheduler: %v", err)
 	}

-	// Setup Prometheus provider
-	if err := setupPrometheusProvider(descheduler, namespacedSharedInformerFactory); err != nil {
-		return nil, nil, fmt.Errorf("failed to setup Prometheus provider: %v", err)
-	}
-
 	// If in dry run mode, replace the descheduler with one using fake client/factory
 	if rs.DryRun {
 		// Create sandbox with resources to mirror from real client
@@ -582,15 +646,10 @@ func bootstrapDescheduler(
 		// TODO(ingvagabund): drop the previous queue
 		// TODO(ingvagabund): stop the previous pod evictor
 		// Replace descheduler with one using fake client/factory
-		descheduler, err = newDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, eventRecorder, kubeClientSandbox.fakeClient(), kubeClientSandbox.fakeSharedInformerFactory(), kubeClientSandbox)
+		descheduler, err = newDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, eventRecorder, kubeClientSandbox.fakeClient(), kubeClientSandbox.fakeSharedInformerFactory(), namespacedSharedInformerFactory, kubeClientSandbox)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to create dry run descheduler: %v", err)
 		}
-
-		// Setup Prometheus provider (with the real shared informer factory as the secret is only read)
-		if err := setupPrometheusProvider(descheduler, namespacedSharedInformerFactory); err != nil {
-			return nil, nil, fmt.Errorf("failed to setup Prometheus provider for the dry run descheduler: %v", err)
-		}
 	}

 	// init is responsible for starting all informer factories, metrics providers
@@ -604,12 +663,12 @@ func bootstrapDescheduler(
 			descheduler.kubeClientSandbox.fakeSharedInformerFactory().WaitForCacheSync(ctx.Done())
 		}
 		sharedInformerFactory.Start(ctx.Done())
-		if metricProviderTokenReconciliation == secretReconciliation {
+		if descheduler.secretBasedPromClientCtrl != nil {
 			namespacedSharedInformerFactory.Start(ctx.Done())
 		}

 		sharedInformerFactory.WaitForCacheSync(ctx.Done())
-		if metricProviderTokenReconciliation == secretReconciliation {
+		if descheduler.secretBasedPromClientCtrl != nil {
 			namespacedSharedInformerFactory.WaitForCacheSync(ctx.Done())
 		}

@@ -629,8 +688,8 @@ func bootstrapDescheduler(
 			}
 		}

-		if metricProviderTokenReconciliation == secretReconciliation {
-			go descheduler.promClientCtrl.runAuthenticationSecretReconciler(ctx)
+		if descheduler.secretBasedPromClientCtrl != nil {
+			go descheduler.secretBasedPromClientCtrl.runAuthenticationSecretReconciler(ctx)
 		}

 		return nil
@@ -641,9 +700,9 @@ func bootstrapDescheduler(
 	}

 	runFnc := func(ctx context.Context) error {
-		if metricProviderTokenReconciliation == inClusterReconciliation {
+		if descheduler.inClusterPromClientCtrl != nil {
 			// Read the sa token and assume it has the sufficient permissions to authenticate
-			if err := descheduler.promClientCtrl.reconcileInClusterSAToken(); err != nil {
+			if err := descheduler.inClusterPromClientCtrl.reconcileInClusterSAToken(); err != nil {
 				return fmt.Errorf("unable to reconcile an in cluster SA token: %v", err)
 			}
 		}
@@ -659,19 +718,6 @@ func bootstrapDescheduler(
 	return descheduler, runFnc, nil
 }

-func prometheusProviderToTokenReconciliation(prometheusProvider *api.MetricsProvider) tokenReconciliation {
-	if prometheusProvider != nil && prometheusProvider.Prometheus != nil && prometheusProvider.Prometheus.URL != "" {
-		if prometheusProvider.Prometheus.AuthToken != nil {
-			// Will get reconciled
-			return secretReconciliation
-		} else {
-			// Use the sa token and assume it has the sufficient permissions to authenticate
-			return inClusterReconciliation
-		}
-	}
-	return noReconciliation
-}
-
 func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer, deschedulerPolicy *api.DeschedulerPolicy, evictionPolicyGroupVersion string) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -692,14 +738,12 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 	defer eventBroadcaster.Shutdown()

 	var namespacedSharedInformerFactory informers.SharedInformerFactory
-
-	prometheusProvider := metricsProviderListToMap(deschedulerPolicy.MetricsProviders)[api.PrometheusMetrics]
-	metricProviderTokenReconciliation := prometheusProviderToTokenReconciliation(prometheusProvider)
-	if metricProviderTokenReconciliation == secretReconciliation {
-		namespacedSharedInformerFactory = informers.NewSharedInformerFactoryWithOptions(rs.Client, 0, informers.WithTransform(trimManagedFields), informers.WithNamespace(prometheusProvider.Prometheus.AuthToken.SecretReference.Namespace))
+	prometheusConfig := getPrometheusConfig(deschedulerPolicy.MetricsProviders)
+	if prometheusConfig != nil && configureSecretPromClientReconciler(prometheusConfig) {
+		namespacedSharedInformerFactory = informers.NewSharedInformerFactoryWithOptions(rs.Client, 0, informers.WithTransform(trimManagedFields), informers.WithNamespace(prometheusConfig.AuthToken.SecretReference.Namespace))
 	}

-	_, runLoop, err := bootstrapDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, metricProviderTokenReconciliation, sharedInformerFactory, namespacedSharedInformerFactory, eventRecorder)
+	_, runLoop, err := bootstrapDescheduler(ctx, rs, deschedulerPolicy, evictionPolicyGroupVersion, sharedInformerFactory, namespacedSharedInformerFactory, eventRecorder)
 	if err != nil {
 		span.AddEvent("Failed to bootstrap a descheduler", trace.WithAttributes(attribute.String("err", err.Error())))
 		return err
@@ -26,6 +26,7 @@ import (
 	fakediscovery "k8s.io/client-go/discovery/fake"
 	"k8s.io/client-go/informers"
 	fakeclientset "k8s.io/client-go/kubernetes/fake"
+	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
 	core "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/cache"
@@ -220,15 +221,13 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate
 	eventBroadcaster, eventRecorder := utils.GetRecorderAndBroadcaster(ctx, client)

 	var namespacedSharedInformerFactory informers.SharedInformerFactory
-
-	prometheusProvider := metricsProviderListToMap(internalDeschedulerPolicy.MetricsProviders)[api.PrometheusMetrics]
-	metricProviderTokenReconciliation := prometheusProviderToTokenReconciliation(prometheusProvider)
-	if metricProviderTokenReconciliation == secretReconciliation {
-		namespacedSharedInformerFactory = informers.NewSharedInformerFactoryWithOptions(rs.Client, 0, informers.WithTransform(trimManagedFields), informers.WithNamespace(prometheusProvider.Prometheus.AuthToken.SecretReference.Namespace))
+	prometheusConfig := getPrometheusConfig(internalDeschedulerPolicy.MetricsProviders)
+	if prometheusConfig != nil && configureSecretPromClientReconciler(prometheusConfig) {
+		namespacedSharedInformerFactory = informers.NewSharedInformerFactoryWithOptions(rs.Client, 0, informers.WithTransform(trimManagedFields), informers.WithNamespace(prometheusConfig.AuthToken.SecretReference.Namespace))
 	}

 	// Always create descheduler with real client/factory first to register all informers
-	descheduler, runFnc, err := bootstrapDescheduler(ctx, rs, internalDeschedulerPolicy, "v1", metricProviderTokenReconciliation, sharedInformerFactory, namespacedSharedInformerFactory, eventRecorder)
+	descheduler, runFnc, err := bootstrapDescheduler(ctx, rs, internalDeschedulerPolicy, "v1", sharedInformerFactory, namespacedSharedInformerFactory, eventRecorder)
 	if err != nil {
 		eventBroadcaster.Shutdown()
 		t.Fatalf("Failed to bootstrap a descheduler: %v", err)
@@ -237,6 +236,10 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate
 	if dryRun {
 		if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
 			for _, obj := range objects {
+				// Only check for nodes - secrets are handled by namespacedSharedInformerFactory
+				if _, ok := obj.(*v1.Node); !ok {
+					continue
+				}
 				exists, err := descheduler.kubeClientSandbox.hasRuntimeObjectInIndexer(obj)
 				if err != nil {
 					return false, err
@@ -261,6 +264,27 @@ func initDescheduler(t *testing.T, ctx context.Context, featureGates featuregate
 	return rs, descheduler, runFnc, client
 }

+func waitForSecretsPropagation(ctx context.Context, secretsLister corev1listers.SecretNamespaceLister, objects []runtime.Object) error {
+	return wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
+		for _, obj := range objects {
+			secret, ok := obj.(*v1.Secret)
+			if !ok {
+				continue
+			}
+			_, err := secretsLister.Get(secret.Name)
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					klog.Infof("Secret %s/%s has not propagated to the indexer", secret.Namespace, secret.Name)
+					return false, nil
+				}
+				return false, err
+			}
+			klog.Infof("Secret %s/%s has propagated to the indexer", secret.Namespace, secret.Name)
+		}
+		return true, nil
+	})
+}
+
 func TestTaintsUpdated(t *testing.T) {
 	initPluginRegistry()

@@ -1323,7 +1347,7 @@ func TestEvictedPodRestorationInDryRun(t *testing.T) {
 	defer eventBroadcaster.Shutdown()

 	// Always create descheduler with real client/factory first to register all informers
-	descheduler, err := newDescheduler(ctxCancel, rs, internalDeschedulerPolicy, "v1", eventRecorder, rs.Client, sharedInformerFactory, nil)
+	descheduler, err := newDescheduler(ctxCancel, rs, internalDeschedulerPolicy, "v1", eventRecorder, rs.Client, sharedInformerFactory, nil, nil)
 	if err != nil {
 		t.Fatalf("Unable to create descheduler instance: %v", err)
 	}
@@ -1338,7 +1362,7 @@ func TestEvictedPodRestorationInDryRun(t *testing.T) {
 	}

 	// Replace descheduler with one using fake client/factory
-	descheduler, err = newDescheduler(ctxCancel, rs, internalDeschedulerPolicy, "v1", eventRecorder, kubeClientSandbox.fakeClient(), kubeClientSandbox.fakeSharedInformerFactory(), kubeClientSandbox)
+	descheduler, err = newDescheduler(ctxCancel, rs, internalDeschedulerPolicy, "v1", eventRecorder, kubeClientSandbox.fakeClient(), kubeClientSandbox.fakeSharedInformerFactory(), nil, kubeClientSandbox)
 	if err != nil {
 		t.Fatalf("Unable to create dry run descheduler instance: %v", err)
 	}
@@ -1552,8 +1576,9 @@ func TestPluginPrometheusClientAccess_Secret(t *testing.T) {
 				t.Logf("Cycle %d: %s", i+1, cycle.name)

 				// Set the descheduler's Prometheus client
-				t.Logf("Setting descheduler.promClientCtrl.promClient from %v to %v", descheduler.promClientCtrl.promClient, cycle.client)
-				descheduler.promClientCtrl.createPrometheusClient = func(url, token string) (promapi.Client, *http.Transport, error) {
+				descheduler.secretBasedPromClientCtrl.mu.Lock()
+				t.Logf("Setting descheduler.secretBasedPromClientCtrl.promClient from %v to %v", descheduler.secretBasedPromClientCtrl.promClient, cycle.client)
+				descheduler.secretBasedPromClientCtrl.createPrometheusClient = func(url, token string) (promapi.Client, *http.Transport, error) {
 					if token != cycle.token {
 						t.Fatalf("Expected token to be %q, got %q", cycle.token, token)
 					}
@@ -1562,6 +1587,7 @@ func TestPluginPrometheusClientAccess_Secret(t *testing.T) {
 					}
 					return cycle.client, &http.Transport{}, nil
 				}
+				descheduler.secretBasedPromClientCtrl.mu.Unlock()

 				if err := cycle.operation(); err != nil {
 					t.Fatalf("operation failed: %v", err)
@@ -1569,7 +1595,7 @@ func TestPluginPrometheusClientAccess_Secret(t *testing.T) {

 				if !cycle.skipWaiting {
 					err := wait.PollUntilContextTimeout(ctx, 50*time.Millisecond, 200*time.Millisecond, true, func(ctx context.Context) (bool, error) {
-						currentPromClient := descheduler.promClientCtrl.prometheusClient()
+						currentPromClient := descheduler.secretBasedPromClientCtrl.prometheusClient()
 						if currentPromClient != cycle.client {
 							t.Logf("Waiting for prometheus client to be set to %v, got %v instead, waiting", cycle.client, currentPromClient)
 							return false, nil
@@ -1590,7 +1616,7 @@ func TestPluginPrometheusClientAccess_Secret(t *testing.T) {
 					t.Fatalf("Unexpected error during running a descheduling cycle: %v", err)
 				}

-				t.Logf("After cycle %d: prometheusClientFromReactor=%v, descheduler.promClientCtrl.promClient=%v", i+1, prometheusClientFromReactor, descheduler.promClientCtrl.promClient)
+				t.Logf("After cycle %d: prometheusClientFromReactor=%v, descheduler.secretBasedPromClientCtrl.promClient=%v", i+1, prometheusClientFromReactor, descheduler.secretBasedPromClientCtrl.prometheusClient())

 				if !newInvoked {
 					t.Fatalf("Expected plugin new to be invoked during cycle %d", i+1)
@@ -1600,7 +1626,7 @@ func TestPluginPrometheusClientAccess_Secret(t *testing.T) {
 					t.Fatalf("Expected deschedule reactor to be invoked during cycle %d", i+1)
 				}

-				verifyAllPrometheusClientsEqual(t, cycle.client, prometheusClientFromReactor, prometheusClientFromPluginNewHandle, descheduler.promClientCtrl.promClient)
+				verifyAllPrometheusClientsEqual(t, cycle.client, prometheusClientFromReactor, prometheusClientFromPluginNewHandle, descheduler.secretBasedPromClientCtrl.prometheusClient())
 			}
 		})
 	}
@@ -1722,11 +1748,12 @@ func TestPluginPrometheusClientAccess_InCluster(t *testing.T) {
 				t.Logf("Cycle %d: %s", i+1, cycle.name)

 				// Set the descheduler's Prometheus client
-				t.Logf("Setting descheduler.promClientCtrl.promClient from %v to %v", descheduler.promClientCtrl.promClient, cycle.client)
-				descheduler.promClientCtrl.inClusterConfig = func() (*rest.Config, error) {
+				descheduler.inClusterPromClientCtrl.mu.Lock()
+				t.Logf("Setting descheduler.inClusterPromClientCtrl.promClient from %v to %v", descheduler.inClusterPromClientCtrl.promClient, cycle.client)
+				descheduler.inClusterPromClientCtrl.inClusterConfig = func() (*rest.Config, error) {
 					return &rest.Config{BearerToken: cycle.token}, nil
 				}
-				descheduler.promClientCtrl.createPrometheusClient = func(url, token string) (promapi.Client, *http.Transport, error) {
+				descheduler.inClusterPromClientCtrl.createPrometheusClient = func(url, token string) (promapi.Client, *http.Transport, error) {
 					if token != cycle.token {
 						t.Errorf("Expected token to be %q, got %q", cycle.token, token)
 					}
@@ -1735,6 +1762,7 @@ func TestPluginPrometheusClientAccess_InCluster(t *testing.T) {
 					}
 					return cycle.client, &http.Transport{}, nil
 				}
+				descheduler.inClusterPromClientCtrl.mu.Unlock()

 				newInvoked = false
 				reactorInvoked = false
@@ -1745,7 +1773,7 @@ func TestPluginPrometheusClientAccess_InCluster(t *testing.T) {
 					t.Fatalf("Unexpected error during running a descheduling cycle: %v", err)
 				}

-				t.Logf("After cycle %d: prometheusClientFromReactor=%v, descheduler.promClientCtrl.promClient=%v", i+1, prometheusClientFromReactor, descheduler.promClientCtrl.promClient)
+				t.Logf("After cycle %d: prometheusClientFromReactor=%v, descheduler.inClusterPromClientCtrl.promClient=%v", i+1, prometheusClientFromReactor, descheduler.inClusterPromClientCtrl.prometheusClient())

 				if !newInvoked {
 					t.Fatalf("Expected plugin new to be invoked during cycle %d", i+1)
@@ -1755,7 +1783,7 @@ func TestPluginPrometheusClientAccess_InCluster(t *testing.T) {
 					t.Fatalf("Expected deschedule reactor to be invoked during cycle %d", i+1)
 				}

-				verifyAllPrometheusClientsEqual(t, cycle.client, prometheusClientFromReactor, prometheusClientFromPluginNewHandle, descheduler.promClientCtrl.promClient)
+				verifyAllPrometheusClientsEqual(t, cycle.client, prometheusClientFromReactor, prometheusClientFromPluginNewHandle, descheduler.inClusterPromClientCtrl.prometheusClient())
 			}
 		})
 	}
@@ -1769,6 +1797,10 @@ func withToken(token string) func(*v1.Secret) {

 func newPrometheusAuthSecret(apply func(*v1.Secret)) *v1.Secret {
 	secret := &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "prom-token",
 			Namespace: "kube-system",
@@ -1787,11 +1819,11 @@ type promClientControllerTestSetup struct {
 	fakeClient                *fakeclientset.Clientset
 	namespacedInformerFactory informers.SharedInformerFactory
 	metricsProviders          map[api.MetricsSource]*api.MetricsProvider
-	ctrl                      *promClientController
+	ctrl                      *secretBasedPromClientController
 	namespace                 string
 }

-func setupPromClientControllerTest(ctx context.Context, objects []runtime.Object, prometheusConfig *api.Prometheus) *promClientControllerTestSetup {
+func setupPromClientControllerTest(ctx context.Context, t *testing.T, objects []runtime.Object, prometheusConfig *api.Prometheus, setNamespacedSharedInformerFactory bool) (*promClientControllerTestSetup, error) {
 	fakeClient := fakeclientset.NewSimpleClientset(objects...)

 	namespace := "default"
@@ -1807,22 +1839,31 @@ func setupPromClientControllerTest(ctx context.Context, objects []runtime.Object
 		Prometheus: prometheusConfig,
 	}})

-	ctrl := newPromClientController(nil, metricsProviders)
-	if prometheusConfig != nil && prometheusConfig.AuthToken != nil && prometheusConfig.AuthToken.SecretReference != nil {
-		ctrl.namespacedSecretsLister = namespacedInformerFactory.Core().V1().Secrets().Lister().Secrets(namespace)
-		namespacedInformerFactory.Core().V1().Secrets().Informer().AddEventHandler(ctrl.eventHandler())
+	ctrl, err := newSecretBasedPromClientController(nil, prometheusConfig, namespacedInformerFactory)
+	if err != nil {
+		return nil, err
 	}

 	namespacedInformerFactory.Start(ctx.Done())
 	namespacedInformerFactory.WaitForCacheSync(ctx.Done())

+	// Wait for secrets to propagate to the informer
+	if len(objects) > 0 && setNamespacedSharedInformerFactory {
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		secretsLister := namespacedInformerFactory.Core().V1().Secrets().Lister().Secrets(namespace)
+		if err := waitForSecretsPropagation(ctx, secretsLister, objects); err != nil {
+			t.Fatalf("secrets did not propagate to the indexer: %v", err)
+		}
+	}
+
 	return &promClientControllerTestSetup{
 		fakeClient:                fakeClient,
 		namespacedInformerFactory: namespacedInformerFactory,
 		metricsProviders:          metricsProviders,
 		ctrl:                      ctrl,
 		namespace:                 namespace,
-	}
+	}, nil
 }

 func newPrometheusConfig() *api.Prometheus {
@@ -1874,6 +1915,59 @@ func TestPromClientControllerSync_InvalidConfig(t *testing.T) {
 			},
 			expectedErr: fmt.Errorf("prometheus metrics source configuration is missing authentication token secret"),
 		},
+		{
+			name: "missing secret reference name",
+			prometheusConfig: &api.Prometheus{
+				URL: prometheusURL,
+				AuthToken: &api.AuthToken{
+					SecretReference: &api.SecretReference{
+						Namespace: "kube-system",
+					},
+				},
+			},
+			expectedErr: fmt.Errorf("prometheus metrics source configuration is missing authentication token secret"),
+		},
+		{
+			name: "missing secret reference namespace",
+			prometheusConfig: &api.Prometheus{
+				URL: prometheusURL,
+				AuthToken: &api.AuthToken{
+					SecretReference: &api.SecretReference{
+						Name: "prom-token",
+					},
+				},
+			},
+			expectedErr: fmt.Errorf("prometheus metrics source configuration is missing authentication token secret"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.TODO())
+			defer cancel()
+			_, err := setupPromClientControllerTest(ctx, t, tc.objects, tc.prometheusConfig, false)
+
+			// Verify error expectations
+			if tc.expectedErr != nil {
+				if err == nil {
+					t.Errorf("Expected error %q but got none", tc.expectedErr)
+				} else if err.Error() != tc.expectedErr.Error() {
+					t.Errorf("Expected error %q but got %q", tc.expectedErr, err.Error())
+				}
+			} else {
+				t.Errorf("Expected an error, got none")
+			}
+		})
+	}
+}
+
+func TestPromClientControllerSync_InvalidSecret(t *testing.T) {
+	testCases := []struct {
+		name             string
+		objects          []runtime.Object
+		prometheusConfig *api.Prometheus
+		expectedErr      error
+	}{
 		{
 			name:             "secret exists but empty token",
 			objects:          []runtime.Object{newPrometheusAuthSecret(withToken(""))},
@@ -1894,10 +1988,13 @@ func TestPromClientControllerSync_InvalidConfig(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.TODO())
 			defer cancel()
-			setup := setupPromClientControllerTest(ctx, tc.objects, tc.prometheusConfig)
+			setup, err := setupPromClientControllerTest(ctx, t, tc.objects, tc.prometheusConfig, true)
+			if err != nil {
+				t.Fatal(err)
+			}

 			// Call sync
-			err := setup.ctrl.sync()
+			err = setup.ctrl.sync()

 			// Verify error expectations
 			if tc.expectedErr != nil {
@@ -1970,18 +2067,21 @@ func TestPromClientControllerSync_ClientCreation(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			for _, setupMode := range []struct {
 				name    string
-				setupFn func(context.Context, *testing.T, []runtime.Object) *promClientController
+				setupFn func(context.Context, *testing.T, []runtime.Object) *secretBasedPromClientController
 			}{
 				{
 					name: "running with prom reconciler directly",
-					setupFn: func(ctx context.Context, t *testing.T, objects []runtime.Object) *promClientController {
-						setup := setupPromClientControllerTest(ctx, objects, newPrometheusConfig())
+					setupFn: func(ctx context.Context, t *testing.T, objects []runtime.Object) *secretBasedPromClientController {
+						setup, err := setupPromClientControllerTest(ctx, t, objects, newPrometheusConfig(), true)
+						if err != nil {
+							t.Fatal(err)
+						}
 						return setup.ctrl
 					},
 				},
 				{
 					name: "running with full descheduler",
-					setupFn: func(ctx context.Context, t *testing.T, objects []runtime.Object) *promClientController {
+					setupFn: func(ctx context.Context, t *testing.T, objects []runtime.Object) *secretBasedPromClientController {
 						deschedulerPolicy := &api.DeschedulerPolicy{
 							MetricsProviders: []api.MetricsProvider{
 								{
@@ -1991,7 +2091,7 @@ func TestPromClientControllerSync_ClientCreation(t *testing.T) {
 							},
 						}
 						_, descheduler, _, _ := initDescheduler(t, ctx, initFeatureGates(), deschedulerPolicy, nil, false, objects...)
-						return descheduler.promClientCtrl
+						return descheduler.secretBasedPromClientCtrl
 					},
 				},
 			} {
@@ -2054,7 +2154,7 @@ func TestPromClientControllerSync_ClientCreation(t *testing.T) {
 					}

 					// Verify promClient cleared when secret not found
-					if tc.expectPreviousTransportCleared && ctrl.promClient != nil {
+					if tc.expectPreviousTransportCleared && ctrl.prometheusClient() != nil {
 						t.Errorf("Expected promClient to be cleared but it wasn't")
 					}

@@ -2075,14 +2175,15 @@ func TestPromClientControllerSync_ClientCreation(t *testing.T) {

 func TestPromClientControllerSync_EventHandler(t *testing.T) {
 	testCases := []struct {
-		name                             string
-		operation                        func(ctx context.Context, fakeClient *fakeclientset.Clientset) error
-		processItem                      bool
-		expectedPromClientSet            bool
-		expectedCreatedClientsCount      int
-		expectedCurrentToken             string
-		expectedPreviousTransportCleared bool
-		expectDifferentClients           bool
+		name                              string
+		operation                         func(ctx context.Context, fakeClient *fakeclientset.Clientset) error
+		processItem                       bool
+		expectedPromClientSet             bool
+		expectedCreatedClientsCount       int
+		expectedCurrentToken              string
+		expectedPreviousTransportCleared  bool
+		expectDifferentClients            bool
+		expectCreatePrometheusClientError bool
 	}{
 		// Check initial conditions
 		{
@@ -2119,15 +2220,56 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {
 			expectedCurrentToken:        "token-2",
 			expectDifferentClients:      true,
 		},
+		{
+			name: "update secret with invalid data",
+			operation: func(ctx context.Context, fakeClient *fakeclientset.Clientset) error {
+				secret := newPrometheusAuthSecret(withToken("token-3"))
+				secret.Data[prometheusAuthTokenSecretKey] = []byte{}
+				_, err := fakeClient.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{})
+				return err
+			},
+			processItem:                 true,
+			expectedPromClientSet:       false,
+			expectedCreatedClientsCount: 2,
+			expectedCurrentToken:        "",
+			expectDifferentClients:      true,
+		},
+		{
+			name: "update secret with valid data",
+			operation: func(ctx context.Context, fakeClient *fakeclientset.Clientset) error {
+				secret := newPrometheusAuthSecret(withToken("token-4"))
+				_, err := fakeClient.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{})
+				return err
+			},
+			processItem:                 true,
+			expectedPromClientSet:       true,
+			expectedCreatedClientsCount: 3,
+			expectedCurrentToken:        "token-4",
+			expectDifferentClients:      true,
+		},
+		{
+			name: "update secret with valid data but createPrometheusClient failing",
+			operation: func(ctx context.Context, fakeClient *fakeclientset.Clientset) error {
+				secret := newPrometheusAuthSecret(withToken("token-5"))
+				_, err := fakeClient.CoreV1().Secrets(secret.Namespace).Update(ctx, secret, metav1.UpdateOptions{})
+				return err
+			},
+			processItem:                       true,
+			expectedPromClientSet:             false,
+			expectedCreatedClientsCount:       3,
+			expectedCurrentToken:              "",
+			expectDifferentClients:            true,
+			expectCreatePrometheusClientError: true,
+		},
 		{
 			name: "delete secret",
 			operation: func(ctx context.Context, fakeClient *fakeclientset.Clientset) error {
-				secret := newPrometheusAuthSecret(withToken("token-2"))
+				secret := newPrometheusAuthSecret(withToken("token-5"))
 				return fakeClient.CoreV1().Secrets(secret.Namespace).Delete(ctx, secret.Name, metav1.DeleteOptions{})
 			},
 			processItem:                      true,
 			expectedPromClientSet:            false,
-			expectedCreatedClientsCount:      2,
+			expectedCreatedClientsCount:      3,
 			expectedCurrentToken:             "",
 			expectedPreviousTransportCleared: true,
 		},
@@ -2135,12 +2277,15 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {

 	for _, setupMode := range []struct {
 		name string
-		init func(t *testing.T, ctx context.Context) (ctrl *promClientController, fakeClient *fakeclientset.Clientset)
+		init func(t *testing.T, ctx context.Context) (ctrl *secretBasedPromClientController, fakeClient *fakeclientset.Clientset)
 	}{
 		{
 			name: "running with prom reconciler directly",
-			init: func(t *testing.T, ctx context.Context) (ctrl *promClientController, fakeClient *fakeclientset.Clientset) {
-				setup := setupPromClientControllerTest(ctx, nil, newPrometheusConfig())
+			init: func(t *testing.T, ctx context.Context) (ctrl *secretBasedPromClientController, fakeClient *fakeclientset.Clientset) {
+				setup, err := setupPromClientControllerTest(ctx, t, nil, newPrometheusConfig(), true)
+				if err != nil {
+					t.Fatal(err)
+				}

 				// Start the reconciler to process queue items
 				go setup.ctrl.runAuthenticationSecretReconciler(ctx)
@@ -2150,7 +2295,7 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {
 		},
 		{
 			name: "running with full descheduler",
-			init: func(t *testing.T, ctx context.Context) (ctrl *promClientController, fakeClient *fakeclientset.Clientset) {
+			init: func(t *testing.T, ctx context.Context) (ctrl *secretBasedPromClientController, fakeClient *fakeclientset.Clientset) {
 				deschedulerPolicy := &api.DeschedulerPolicy{
 					MetricsProviders: []api.MetricsProvider{
 						{
@@ -2163,7 +2308,7 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {
 				_, descheduler, _, client := initDescheduler(t, ctx, initFeatureGates(), deschedulerPolicy, nil, false)
 				// The reconciler is already started by initDescheduler via bootstrapDescheduler

-				return descheduler.promClientCtrl, client
+				return descheduler.secretBasedPromClientCtrl, client
 			},
 		},
 	} {
@@ -2176,16 +2321,22 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {
 			// Track created clients to verify different instances
 			var createdClients []promapi.Client
 			var createdClientsMu sync.Mutex
-			ctrl.createPrometheusClient = func(url, token string) (promapi.Client, *http.Transport, error) {
-				client := &mockPrometheusClient{name: "client-" + token}
-				createdClientsMu.Lock()
-				createdClients = append(createdClients, client)
-				createdClientsMu.Unlock()
-				return client, &http.Transport{}, nil
-			}

 			for _, tc := range testCases {
 				t.Run(tc.name, func(t *testing.T) {
+					ctrl.mu.Lock()
+					ctrl.createPrometheusClient = func(url, token string) (promapi.Client, *http.Transport, error) {
+						if tc.expectCreatePrometheusClientError {
+							return nil, &http.Transport{}, fmt.Errorf("error creating a prometheus client")
+						}
+						client := &mockPrometheusClient{name: "client-" + token}
+						createdClientsMu.Lock()
+						createdClients = append(createdClients, client)
+						createdClientsMu.Unlock()
+						return client, &http.Transport{}, nil
+					}
+					ctrl.mu.Unlock()
+
 					if err := tc.operation(ctx, fakeClient); err != nil {
 						t.Fatalf("Failed to execute operation: %v", err)
 					}
@@ -2193,13 +2344,20 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {
 					if tc.processItem {
 						// Wait for event to be processed by the reconciler
 						err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 2*time.Second, true, func(ctx context.Context) (bool, error) {
-							// Check if all expected conditions are met
+							// Check if all expected conditions are met (with mutex protection)
+							ctrl.mu.RLock()
+							promClient := ctrl.promClient
+							currentToken := ctrl.currentPrometheusAuthToken
+							previousTransport := ctrl.previousPrometheusClientTransport
+							ctrl.mu.RUnlock()
+
+							t.Logf("promClient: %v\n", promClient)
 							if tc.expectedPromClientSet {
-								if ctrl.promClient == nil {
+								if promClient == nil {
 									return false, nil
 								}
 							} else {
-								if ctrl.promClient != nil {
+								if promClient != nil {
 									return false, nil
 								}
 							}
@@ -2207,16 +2365,17 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {
 							createdClientsMu.Lock()
 							createdClientsLen := len(createdClients)
 							createdClientsMu.Unlock()
+							t.Logf("createdClientsLen: %v\n", createdClientsLen)
 							if createdClientsLen != tc.expectedCreatedClientsCount {
 								return false, nil
 							}
-
-							if ctrl.currentPrometheusAuthToken != tc.expectedCurrentToken {
+							t.Logf("currentToken: %v\n", currentToken)
+							if currentToken != tc.expectedCurrentToken {
 								return false, nil
 							}
-
+							t.Logf("previousTransport: %v\n", previousTransport)
 							if tc.expectedPreviousTransportCleared {
-								if ctrl.previousPrometheusClientTransport != nil {
+								if previousTransport != nil {
 									return false, nil
 								}
 							}
@@ -2234,12 +2393,13 @@ func TestPromClientControllerSync_EventHandler(t *testing.T) {

 					// Validate post-conditions
 					if tc.expectedPromClientSet {
-						if ctrl.promClient == nil {
+						if ctrl.prometheusClient() == nil {
 							t.Error("Expected prometheus client to be set, but it was nil")
 						}
 					} else {
-						if ctrl.promClient != nil {
-							t.Errorf("Expected prometheus client to be nil, but got: %v", ctrl.promClient)
+						promClient := ctrl.prometheusClient()
+						if promClient != nil {
+							t.Errorf("Expected prometheus client to be nil, but got: %v", promClient)
 						}
 					}

@@ -2320,7 +2480,7 @@ func TestReconcileInClusterSAToken(t *testing.T) {
 			},
 			expectedErr:                    fmt.Errorf("unable to create a prometheus client: failed to create client"),
 			expectClientCreated:            false,
-			expectCurrentToken:             "old-token",
+			expectCurrentToken:             "",
 			expectPreviousTransportCleared: false,
 			expectPromClientCleared:        false,
 		},
@@ -2374,20 +2534,15 @@ func TestReconcileInClusterSAToken(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			for _, setupMode := range []struct {
 				name string
-				init func(t *testing.T) *promClientController
+				init func(t *testing.T) *inClusterPromClientController
 			}{
 				{
 					name: "running with prom reconciler directly",
-					init: func(t *testing.T) *promClientController {
-						return &promClientController{
+					init: func(t *testing.T) *inClusterPromClientController {
+						return &inClusterPromClientController{
 							currentPrometheusAuthToken: tc.currentAuthToken,
-							metricsProviders: map[api.MetricsSource]*api.MetricsProvider{
-								api.PrometheusMetrics: {
-									Source: api.PrometheusMetrics,
-									Prometheus: &api.Prometheus{
-										URL: prometheusURL,
-									},
-								},
+							prometheusConfig: &api.Prometheus{
+								URL: prometheusURL,
 							},
 							inClusterConfig: tc.inClusterConfigFunc,
 						}
@@ -2395,7 +2550,7 @@ func TestReconcileInClusterSAToken(t *testing.T) {
 				},
 				{
 					name: "running with full descheduler",
-					init: func(t *testing.T) *promClientController {
+					init: func(t *testing.T) *inClusterPromClientController {
 						ctx := context.Background()
 						prometheusConfig := &api.Prometheus{
 							URL: prometheusURL,
@@ -2410,10 +2565,11 @@ func TestReconcileInClusterSAToken(t *testing.T) {
 						}
 						_, descheduler, _, _ := initDescheduler(t, ctx, initFeatureGates(), deschedulerPolicy, nil, false)

-						// Override the fields needed for this test
-						descheduler.promClientCtrl.currentPrometheusAuthToken = tc.currentAuthToken
-						descheduler.promClientCtrl.inClusterConfig = tc.inClusterConfigFunc
-						return descheduler.promClientCtrl
+						// Override the fields needed for this test (no need for a mutex
+						// since reconcileInClusterSAToken gets to run later)
+						descheduler.inClusterPromClientCtrl.currentPrometheusAuthToken = tc.currentAuthToken
+						descheduler.inClusterPromClientCtrl.inClusterConfig = tc.inClusterConfigFunc
+						return descheduler.inClusterPromClientCtrl
 					},
 				},
 			} {