From 8bc66c937d08cc4e62380b6e853108b690a345d2 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 16 Aug 2023 21:15:09 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20don=E2=80=99t=20hold=20node=20lock=20if?= =?UTF-8?q?=20reboot=20is=20blocked=20(#819)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jack Francis --- cmd/kured/main.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 69d55a0..09f9913 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -737,7 +737,6 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s preferNoScheduleTaint.Disable() continue } - log.Infof("Reboot required") node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{}) if err != nil { @@ -761,12 +760,6 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s } } - if !holding(lock, &nodeMeta, concurrency > 1) && !acquire(lock, &nodeMeta, TTL, concurrency) { - // Prefer to not schedule pods onto this node to avoid draing the same pod multiple times. - preferNoScheduleTaint.Enable() - continue - } - var blockCheckers []RebootBlocker if prometheusURL != "" { blockCheckers = append(blockCheckers, PrometheusBlockingChecker{promClient: promClient, filter: alertFilter, firingOnly: alertFiringOnly, filterMatchOnly: alertFilterMatchOnly}) @@ -775,7 +768,16 @@ func rebootAsRequired(nodeID string, rebootCommand []string, sentinelCommand []s blockCheckers = append(blockCheckers, KubernetesBlockingChecker{client: client, nodename: nodeID, filter: podSelectors}) } + var rebootRequiredBlockCondition string if rebootBlocked(blockCheckers...) { + rebootRequiredBlockCondition = ", but blocked at this time" + continue + } + log.Infof("Reboot required%s", rebootRequiredBlockCondition) + + if !holding(lock, &nodeMeta, concurrency > 1) && !acquire(lock, &nodeMeta, TTL, concurrency) { + // Prefer to not schedule pods onto this node to avoid draing the same pod multiple times. + preferNoScheduleTaint.Enable() continue }