From 65292983f2ed34171345d586d017c2a39d21c11a Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Thu, 17 Dec 2020 13:36:02 -0500 Subject: [PATCH 1/6] Add force-reboot after force-timeout duration has been exceeded --- README.md | 3 +++ charts/kured/Chart.yaml | 2 +- charts/kured/README.md | 3 +++ charts/kured/templates/daemonset.yaml | 9 +++++++++ charts/kured/values.minikube.yaml | 3 +++ charts/kured/values.yaml | 3 +++ cmd/kured/main.go | 22 ++++++++++++++++++++-- 7 files changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 44836ac..5eaba7f 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,12 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots + --drain-grace-period int period of time to wait for node drain in seconds (default -1) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") + --force-reboot bool force a reboot even if the drain is still running (default false) + --force-timeout duration total drain timeout, only applies when force-reboot is enabled (default 30m) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/charts/kured/Chart.yaml b/charts/kured/Chart.yaml index 1f3c240..0be27a2 100644 --- a/charts/kured/Chart.yaml +++ b/charts/kured/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: "1.6.1" description: A Helm chart for kured name: kured -version: 2.4.2 +version: 2.4.3 home: https://github.com/weaveworks/kured maintainers: - name: ckotzbauer diff --git a/charts/kured/README.md b/charts/kured/README.md index 3b4b7b8..cf43f20 100644 --- a/charts/kured/README.md +++ b/charts/kured/README.md @@ -50,6 +50,7 @@ The following changes have been made compared to the stable chart: | `configuration.endTime` | cli-parameter `--end-time` | `""` | | `configuration.lockAnnotation` | cli-parameter `--lock-annotation` | `""` | | `configuration.period` | cli-parameter `--period` | `""` | +| `configuration.drainGracePeriod` | cli-parameter `--drain-grace-period` | `""` | | `configuration.prometheusUrl` | cli-parameter `--prometheus-url` | `""` | | `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days` | `[]` | | `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel` | `""` | @@ -61,6 +62,8 @@ The following changes have been made compared to the stable chart: | `configuration.startTime` | cli-parameter `--start-time` | `""` | | `configuration.timeZone` | cli-parameter `--time-zone` | `""` | | `configuration.annotateNodes` | cli-parameter `--annotate-nodes` | `false` | +| `configuration.forceReboot` | cli-parameter `--force-reboot` | `false` | +| `configuration.forceTimeout` | cli-parameter `--force-timeout` | `""` | | `rbac.create` | Create RBAC roles | `true` | | `serviceAccount.create` | Create a service account | `true` | | `serviceAccount.name` | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname) | diff --git a/charts/kured/templates/daemonset.yaml b/charts/kured/templates/daemonset.yaml index c2a6b97..4ee9f69 100644 --- a/charts/kured/templates/daemonset.yaml +++ b/charts/kured/templates/daemonset.yaml @@ -70,6 +70,9 @@ spec: {{- if .Values.configuration.period }} - --period={{ .Values.configuration.period }} {{- end }} + {{- if .Values.configuration.drainGracePeriod }} + - --drain-grace-period={{ .Values.configuration.drainGracePeriod }} + {{- end }} {{- if .Values.configuration.prometheusUrl }} - --prometheus-url={{ .Values.configuration.prometheusUrl }} {{- end }} @@ -103,6 +106,12 @@ spec: {{- if .Values.configuration.annotateNodes }} - --annotate-nodes={{ .Values.configuration.annotateNodes }} {{- end }} + {{- if .Values.configuration.forceReboot }} + - --force-reboot + {{- end }} + {{- if .Values.configuration.forceTimeout }} + - --force-timeout={{ .Values.configuration.forceTimeout }} + {{- end }} {{- range $key, $value := .Values.extraArgs }} {{- if $value }} - --{{ $key }}={{ $value }} diff --git a/charts/kured/values.minikube.yaml b/charts/kured/values.minikube.yaml index cf4a554..dcc5297 100644 --- a/charts/kured/values.minikube.yaml +++ b/charts/kured/values.minikube.yaml @@ -9,6 +9,7 @@ configuration: # endTime: "" # only reboot before this time of day (default "23:59") # lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "1m" # reboot check period (default 1h0m0s) + # drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) # prometheusUrl: "" # Prometheus instance to probe for active alerts # rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) # rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -20,3 +21,5 @@ configuration: # startTime: "" # only reboot after this time of day (default "0:00") # timeZone: "" # time-zone to use (valid zones from "time" golang package) # annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations + # forceReboot: false # force a reboot even if the drain is still running (default false) + # forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index eb0022b..29f70c9 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -28,6 +28,7 @@ configuration: endTime: "" # only reboot before this time of day (default "23:59") lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "" # reboot check period (default 1h0m0s) + drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) prometheusUrl: "" # Prometheus instance to probe for active alerts rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -39,6 +40,8 @@ configuration: startTime: "" # only reboot after this time of day (default "0:00") timeZone: "" # time-zone to use (valid zones from "time" golang package) annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations + forceReboot: false # force a reboot even if the drain is still running (default false) + forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) rbac: create: true diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 3810145..ddfe09f 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,7 +38,11 @@ var ( version = "unreleased" // Command line flags + forceDrain bool + forceReboot bool + forceTimeout time.Duration period time.Duration + drainGracePeriod int dsNamespace string dsName string lockAnnotation string @@ -91,6 +95,12 @@ func main() { PreRun: flagCheck, Run: root} + rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, + "enable/disable force reboot") + rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, + "drain grace period in seconds") + rootCmd.PersistentFlags().DurationVar(&forceTimeout, "force-timeout", time.Minute*30, + "total timeout which only applies when force-reboot is set to true") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", @@ -333,19 +343,27 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { drainer := &kubectldrain.Helper{ Client: client, - GracePeriodSeconds: -1, + Ctx: context.Background(), + GracePeriodSeconds: drainGracePeriod, Force: true, DeleteEmptyDirData: true, IgnoreAllDaemonSets: true, ErrOut: os.Stderr, Out: os.Stdout, } + if forceReboot { + drainer.Timeout = forceTimeout + } + if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { log.Fatalf("Error cordonning %s: %v", nodename, err) } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { - log.Fatalf("Error draining %s: %v", nodename, err) + if !forceReboot { + log.Fatalf("Error draining %s: %v", nodename, err) + } + log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err) } } From 390f6e9f9930b6a805baef6a2dbeb7f5544b0a2f Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 8 Mar 2021 16:14:19 -0800 Subject: [PATCH 2/6] chore: retry daemonset get operations --- pkg/daemonsetlock/daemonsetlock.go | 41 ++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/pkg/daemonsetlock/daemonsetlock.go b/pkg/daemonsetlock/daemonsetlock.go index eb038be..631d0df 100644 --- a/pkg/daemonsetlock/daemonsetlock.go +++ b/pkg/daemonsetlock/daemonsetlock.go @@ -6,11 +6,18 @@ import ( "fmt" "time" + v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) +const ( + k8sAPICallRetrySleep = 5 * time.Second // How much time to wait in between retrying a k8s API call + k8sAPICallRetryTimeout = 5 * time.Minute // How long to wait until we determine that the k8s API is definitively unavailable +) + // DaemonSetLock holds all necessary information to do actions // on the kured ds which holds lock info through annotations. type DaemonSetLock struct { @@ -34,11 +41,11 @@ func New(client *kubernetes.Clientset, nodeID, namespace, name, annotation strin } // Acquire attempts to annotate the kured daemonset with lock info from instantiated DaemonSetLock using client-go -func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acquired bool, owner string, err error) { +func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (bool, string, error) { for { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return false, "", err + return false, "", fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -78,10 +85,10 @@ func (dsl *DaemonSetLock) Acquire(metadata interface{}, TTL time.Duration) (acqu } // Test attempts to check the kured daemonset lock status (existence, expiry) from instantiated DaemonSetLock using client-go -func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) +func (dsl *DaemonSetLock) Test(metadata interface{}) (bool, error) { + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return false, err + return false, fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -102,9 +109,9 @@ func (dsl *DaemonSetLock) Test(metadata interface{}) (holding bool, err error) { // Release attempts to remove the lock data from the kured ds annotations using client-go func (dsl *DaemonSetLock) Release() error { for { - ds, err := dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(context.TODO(), dsl.name, metav1.GetOptions{}) + ds, err := dsl.GetDaemonSet(k8sAPICallRetrySleep, k8sAPICallRetryTimeout) if err != nil { - return err + return fmt.Errorf("timed out trying to get daemonset %s in namespace %s: %w", dsl.name, dsl.namespace, err) } valueString, exists := ds.ObjectMeta.Annotations[dsl.annotation] @@ -137,6 +144,24 @@ func (dsl *DaemonSetLock) Release() error { } } +// GetDaemonSet returns the named DaemonSet resource from the DaemonSetLock's configured client +func (dsl *DaemonSetLock) GetDaemonSet(sleep, timeout time.Duration) (*v1.DaemonSet, error) { + var ds *v1.DaemonSet + var lastError error + err := wait.PollImmediate(sleep, timeout, func() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + if ds, lastError = dsl.client.AppsV1().DaemonSets(dsl.namespace).Get(ctx, dsl.name, metav1.GetOptions{}); lastError != nil { + return false, nil + } + return true, nil + }) + if err != nil { + return nil, fmt.Errorf("Timed out trying to get daemonset %s in namespace %s: %v", dsl.name, dsl.namespace, lastError) + } + return ds, nil +} + func ttlExpired(created time.Time, ttl time.Duration) bool { if ttl > 0 && time.Since(created) >= ttl { return true From 8db565051047dcd6857bd6c6237554661bf24d54 Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Wed, 7 Apr 2021 12:53:19 -0500 Subject: [PATCH 3/6] Refactor force-drain to be a drain-timeout in general --- README.md | 4 ++-- charts/kured/Chart.yaml | 2 +- charts/kured/README.md | 3 --- charts/kured/templates/daemonset.yaml | 9 --------- charts/kured/values.minikube.yaml | 3 --- charts/kured/values.yaml | 3 --- cmd/kured/main.go | 15 ++++++--------- 7 files changed, 9 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 5eaba7f..fd9f6ea 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,12 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots - --drain-grace-period int period of time to wait for node drain in seconds (default -1) + --drain-grace-period int grace period of time for pods to wait for the node drain in seconds (default -1) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") --force-reboot bool force a reboot even if the drain is still running (default false) - --force-timeout duration total drain timeout, only applies when force-reboot is enabled (default 30m) + --drain-timeout duration timeout after which the drain is aborted (default 30m) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/charts/kured/Chart.yaml b/charts/kured/Chart.yaml index 0be27a2..1f3c240 100644 --- a/charts/kured/Chart.yaml +++ b/charts/kured/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: "1.6.1" description: A Helm chart for kured name: kured -version: 2.4.3 +version: 2.4.2 home: https://github.com/weaveworks/kured maintainers: - name: ckotzbauer diff --git a/charts/kured/README.md b/charts/kured/README.md index cf43f20..3b4b7b8 100644 --- a/charts/kured/README.md +++ b/charts/kured/README.md @@ -50,7 +50,6 @@ The following changes have been made compared to the stable chart: | `configuration.endTime` | cli-parameter `--end-time` | `""` | | `configuration.lockAnnotation` | cli-parameter `--lock-annotation` | `""` | | `configuration.period` | cli-parameter `--period` | `""` | -| `configuration.drainGracePeriod` | cli-parameter `--drain-grace-period` | `""` | | `configuration.prometheusUrl` | cli-parameter `--prometheus-url` | `""` | | `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days` | `[]` | | `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel` | `""` | @@ -62,8 +61,6 @@ The following changes have been made compared to the stable chart: | `configuration.startTime` | cli-parameter `--start-time` | `""` | | `configuration.timeZone` | cli-parameter `--time-zone` | `""` | | `configuration.annotateNodes` | cli-parameter `--annotate-nodes` | `false` | -| `configuration.forceReboot` | cli-parameter `--force-reboot` | `false` | -| `configuration.forceTimeout` | cli-parameter `--force-timeout` | `""` | | `rbac.create` | Create RBAC roles | `true` | | `serviceAccount.create` | Create a service account | `true` | | `serviceAccount.name` | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname) | diff --git a/charts/kured/templates/daemonset.yaml b/charts/kured/templates/daemonset.yaml index 4ee9f69..c2a6b97 100644 --- a/charts/kured/templates/daemonset.yaml +++ b/charts/kured/templates/daemonset.yaml @@ -70,9 +70,6 @@ spec: {{- if .Values.configuration.period }} - --period={{ .Values.configuration.period }} {{- end }} - {{- if .Values.configuration.drainGracePeriod }} - - --drain-grace-period={{ .Values.configuration.drainGracePeriod }} - {{- end }} {{- if .Values.configuration.prometheusUrl }} - --prometheus-url={{ .Values.configuration.prometheusUrl }} {{- end }} @@ -106,12 +103,6 @@ spec: {{- if .Values.configuration.annotateNodes }} - --annotate-nodes={{ .Values.configuration.annotateNodes }} {{- end }} - {{- if .Values.configuration.forceReboot }} - - --force-reboot - {{- end }} - {{- if .Values.configuration.forceTimeout }} - - --force-timeout={{ .Values.configuration.forceTimeout }} - {{- end }} {{- range $key, $value := .Values.extraArgs }} {{- if $value }} - --{{ $key }}={{ $value }} diff --git a/charts/kured/values.minikube.yaml b/charts/kured/values.minikube.yaml index dcc5297..cf4a554 100644 --- a/charts/kured/values.minikube.yaml +++ b/charts/kured/values.minikube.yaml @@ -9,7 +9,6 @@ configuration: # endTime: "" # only reboot before this time of day (default "23:59") # lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "1m" # reboot check period (default 1h0m0s) - # drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) # prometheusUrl: "" # Prometheus instance to probe for active alerts # rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) # rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -21,5 +20,3 @@ configuration: # startTime: "" # only reboot after this time of day (default "0:00") # timeZone: "" # time-zone to use (valid zones from "time" golang package) # annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations - # forceReboot: false # force a reboot even if the drain is still running (default false) - # forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index 29f70c9..eb0022b 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -28,7 +28,6 @@ configuration: endTime: "" # only reboot before this time of day (default "23:59") lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "" # reboot check period (default 1h0m0s) - drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) prometheusUrl: "" # Prometheus instance to probe for active alerts rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -40,8 +39,6 @@ configuration: startTime: "" # only reboot after this time of day (default "0:00") timeZone: "" # time-zone to use (valid zones from "time" golang package) annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations - forceReboot: false # force a reboot even if the drain is still running (default false) - forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) rbac: create: true diff --git a/cmd/kured/main.go b/cmd/kured/main.go index ddfe09f..7407b09 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,9 +38,8 @@ var ( version = "unreleased" // Command line flags - forceDrain bool forceReboot bool - forceTimeout time.Duration + drainTimeout time.Duration period time.Duration drainGracePeriod int dsNamespace string @@ -96,11 +95,11 @@ func main() { Run: root} rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, - "enable/disable force reboot") + "force a reboot even if the drain is still running (default false)") rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, - "drain grace period in seconds") - rootCmd.PersistentFlags().DurationVar(&forceTimeout, "force-timeout", time.Minute*30, - "total timeout which only applies when force-reboot is set to true") + "grace period of time for pods to wait for the node drain in seconds (default -1)") + rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", time.Minute*30, + "timeout after which the drain is aborted (default: 30m)") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", @@ -350,9 +349,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { IgnoreAllDaemonSets: true, ErrOut: os.Stderr, Out: os.Stdout, - } - if forceReboot { - drainer.Timeout = forceTimeout + Timeout: drainTimeout, } if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { From 2400f34cc0270cdb99134101fbe694c05c5a960e Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Wed, 7 Apr 2021 14:58:21 -0500 Subject: [PATCH 4/6] Don't panic if the cordon fails and force-reboot is true --- cmd/kured/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 7407b09..71031ab 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -353,7 +353,10 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { } if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { - log.Fatalf("Error cordonning %s: %v", nodename, err) + if !forceReboot { + log.Fatalf("Error cordonning %s: %v", nodename, err) + } + log.Errorf("Error cordonning %s: %v, continuing with reboot anyway", nodename, err) } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { From 5a86ef40e87b784ade4aed0ed1b9713da38158d2 Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Wed, 7 Apr 2021 17:17:33 -0500 Subject: [PATCH 5/6] Update the default drain timeout to be infinite --- cmd/kured/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 71031ab..7335f41 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -98,8 +98,8 @@ func main() { "force a reboot even if the drain is still running (default false)") rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, "grace period of time for pods to wait for the node drain in seconds (default -1)") - rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", time.Minute*30, - "timeout after which the drain is aborted (default: 30m)") + rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0, + "timeout after which the drain is aborted (default: 0, infinite time)") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", From 25dcf3cb1286347199693b54844e78cb5944c54e Mon Sep 17 00:00:00 2001 From: Cameron McAvoy Date: Thu, 8 Apr 2021 09:49:44 -0500 Subject: [PATCH 6/6] Expose SkipWaitForDeleteTimeoutSeconds and explicitly return when cordonning fails --- README.md | 7 +++-- cmd/kured/main.go | 70 +++++++++++++++++++++++++---------------------- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index fd9f6ea..d5c9959 100644 --- a/README.md +++ b/README.md @@ -83,12 +83,13 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots - --drain-grace-period int grace period of time for pods to wait for the node drain in seconds (default -1) + --drain-grace-period int time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1) + --skip-wait-for-delete-timeout int when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") - --force-reboot bool force a reboot even if the drain is still running (default false) - --drain-timeout duration timeout after which the drain is aborted (default 30m) + --force-reboot bool force a reboot even if the drain is still running (default: false) + --drain-timeout duration timeout after which the drain is aborted (default: 0, infinite time) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 7335f41..45b71f8 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,27 +38,28 @@ var ( version = "unreleased" // Command line flags - forceReboot bool - drainTimeout time.Duration - period time.Duration - drainGracePeriod int - dsNamespace string - dsName string - lockAnnotation string - lockTTL time.Duration - prometheusURL string - preferNoScheduleTaintName string - alertFilter *regexp.Regexp - rebootSentinelFile string - rebootSentinelCommand string - notifyURL string - slackHookURL string - slackUsername string - slackChannel string - messageTemplateDrain string - messageTemplateReboot string - podSelectors []string - rebootCommand string + forceReboot bool + drainTimeout time.Duration + period time.Duration + drainGracePeriod int + skipWaitForDeleteTimeoutSeconds int + dsNamespace string + dsName string + lockAnnotation string + lockTTL time.Duration + prometheusURL string + preferNoScheduleTaintName string + alertFilter *regexp.Regexp + rebootSentinelFile string + rebootSentinelCommand string + notifyURL string + slackHookURL string + slackUsername string + slackChannel string + messageTemplateDrain string + messageTemplateReboot string + podSelectors []string + rebootCommand string rebootDays []string rebootStart string @@ -95,9 +96,11 @@ func main() { Run: root} rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, - "force a reboot even if the drain is still running (default false)") + "force a reboot even if the drain is still running (default: false)") rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, - "grace period of time for pods to wait for the node drain in seconds (default -1)") + "time in seconds given to each pod to terminate gracefully, if negative, the default value specified in the pod will be used (default: -1)") + rootCmd.PersistentFlags().IntVar(&skipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", 0, + "when seconds is greater than zero, skip waiting for the pods whose deletion timestamp is older than N seconds while draining a node (default: 0)") rootCmd.PersistentFlags().DurationVar(&drainTimeout, "drain-timeout", 0, "timeout after which the drain is aborted (default: 0, infinite time)") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, @@ -341,15 +344,16 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { } drainer := &kubectldrain.Helper{ - Client: client, - Ctx: context.Background(), - GracePeriodSeconds: drainGracePeriod, - Force: true, - DeleteEmptyDirData: true, - IgnoreAllDaemonSets: true, - ErrOut: os.Stderr, - Out: os.Stdout, - Timeout: drainTimeout, + Client: client, + Ctx: context.Background(), + GracePeriodSeconds: drainGracePeriod, + SkipWaitForDeleteTimeoutSeconds: skipWaitForDeleteTimeoutSeconds, + Force: true, + DeleteEmptyDirData: true, + IgnoreAllDaemonSets: true, + ErrOut: os.Stderr, + Out: os.Stdout, + Timeout: drainTimeout, } if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { @@ -357,6 +361,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { log.Fatalf("Error cordonning %s: %v", nodename, err) } log.Errorf("Error cordonning %s: %v, continuing with reboot anyway", nodename, err) + return } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { @@ -364,6 +369,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { log.Fatalf("Error draining %s: %v", nodename, err) } log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err) + return } }