From 37fe4493e73fe9d7111d4f2424964a734c7ffba3 Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Mon, 12 May 2025 11:51:35 +0300 Subject: [PATCH] Fix HA init server scaling (#333) * Fix HA init server scaling Signed-off-by: galal-hussein * increase timeout in e2e test Signed-off-by: galal-hussein --------- Signed-off-by: galal-hussein --- pkg/controller/cluster/server/server.go | 14 ++++++++++++++ pkg/controller/cluster/server/template.go | 8 ++++---- tests/cluster_test.go | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 75e67e09..3d265116 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -203,6 +203,20 @@ func (s *Server) podSpec(image, name string, persistent bool, startupCmd string) }, }, } + podSpec.Containers[0].LivenessProbe = &v1.Probe{ + InitialDelaySeconds: 10, + FailureThreshold: 3, + PeriodSeconds: 3, + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{ + "sh", + "-c", + `grep -q "rejoin the cluster" /var/log/k3s.log && exit 1 || exit 0`, + }, + }, + }, + } // start the pod unprivileged in shared mode if s.mode == agent.VirtualNodeMode { podSpec.Containers[0].SecurityContext = &v1.SecurityContext{ diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index 42fae023..b472f68d 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -3,14 +3,14 @@ package server var singleServerTemplate string = ` if [ -d "{{.ETCD_DIR}}" ]; then # if directory exists then it means its not an initial run - /bin/k3s server --cluster-reset --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}} + /bin/k3s server --cluster-reset --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}} 2>&1 | tee /var/log/k3s.log fi rm -f /var/lib/rancher/k3s/server/db/reset-flag -/bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}}` +/bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}} 2>&1 | tee /var/log/k3s.log` var HAServerTemplate string = ` if [ ${POD_NAME: -1} == 0 ] && [ ! -d "{{.ETCD_DIR}}" ]; then - /bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}} + /bin/k3s server --config {{.INIT_CONFIG}} {{.EXTRA_ARGS}} 2>&1 | tee /var/log/k3s.log else - /bin/k3s server --config {{.SERVER_CONFIG}} {{.EXTRA_ARGS}} + /bin/k3s server --config {{.SERVER_CONFIG}} {{.EXTRA_ARGS}} 2>&1 | tee /var/log/k3s.log fi` diff --git a/tests/cluster_test.go b/tests/cluster_test.go index dc4acf29..1a11ea09 100644 --- a/tests/cluster_test.go +++ b/tests/cluster_test.go @@ -165,7 +165,7 @@ var _ = When("a dynamic cluster is installed", func() { Expect(len(serverPods.Items)).To(Equal(1)) return serverPods.Items[0].DeletionTimestamp }). - WithTimeout(30 * time.Second). + WithTimeout(60 * time.Second). WithPolling(time.Second * 5). Should(BeNil())