diff --git a/README.md b/README.md index 44836ac..5eaba7f 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,12 @@ The following arguments can be passed to kured via the daemonset pod template: Flags: --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots + --drain-grace-period int period of time to wait for node drain in seconds (default -1) --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") --end-time string schedule reboot only before this time of day (default "23:59:59") + --force-reboot bool force a reboot even if the drain is still running (default false) + --force-timeout duration total drain timeout, only applies when force-reboot is enabled (default 30m) -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) diff --git a/charts/kured/Chart.yaml b/charts/kured/Chart.yaml index 1f3c240..0be27a2 100644 --- a/charts/kured/Chart.yaml +++ b/charts/kured/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: "1.6.1" description: A Helm chart for kured name: kured -version: 2.4.2 +version: 2.4.3 home: https://github.com/weaveworks/kured maintainers: - name: ckotzbauer diff --git a/charts/kured/README.md b/charts/kured/README.md index 3b4b7b8..cf43f20 100644 --- a/charts/kured/README.md +++ b/charts/kured/README.md @@ -50,6 +50,7 @@ The following changes have been made compared to the stable chart: | `configuration.endTime` | cli-parameter `--end-time` | `""` | | `configuration.lockAnnotation` | cli-parameter `--lock-annotation` | `""` | | `configuration.period` | cli-parameter `--period` | `""` | +| `configuration.drainGracePeriod` | cli-parameter `--drain-grace-period` | `""` | | `configuration.prometheusUrl` | cli-parameter `--prometheus-url` | `""` | | `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days` | `[]` | | `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel` | `""` | @@ -61,6 +62,8 @@ The following changes have been made compared to the stable chart: | `configuration.startTime` | cli-parameter `--start-time` | `""` | | `configuration.timeZone` | cli-parameter `--time-zone` | `""` | | `configuration.annotateNodes` | cli-parameter `--annotate-nodes` | `false` | +| `configuration.forceReboot` | cli-parameter `--force-reboot` | `false` | +| `configuration.forceTimeout` | cli-parameter `--force-timeout` | `""` | | `rbac.create` | Create RBAC roles | `true` | | `serviceAccount.create` | Create a service account | `true` | | `serviceAccount.name` | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname) | diff --git a/charts/kured/templates/daemonset.yaml b/charts/kured/templates/daemonset.yaml index c2a6b97..4ee9f69 100644 --- a/charts/kured/templates/daemonset.yaml +++ b/charts/kured/templates/daemonset.yaml @@ -70,6 +70,9 @@ spec: {{- if .Values.configuration.period }} - --period={{ .Values.configuration.period }} {{- end }} + {{- if .Values.configuration.drainGracePeriod }} + - --drain-grace-period={{ .Values.configuration.drainGracePeriod }} + {{- end }} {{- if .Values.configuration.prometheusUrl }} - --prometheus-url={{ .Values.configuration.prometheusUrl }} {{- end }} @@ -103,6 +106,12 @@ spec: {{- if .Values.configuration.annotateNodes }} - --annotate-nodes={{ .Values.configuration.annotateNodes }} {{- end }} + {{- if .Values.configuration.forceReboot }} + - --force-reboot + {{- end }} + {{- if .Values.configuration.forceTimeout }} + - --force-timeout={{ .Values.configuration.forceTimeout }} + {{- end }} {{- range $key, $value := .Values.extraArgs }} {{- if $value }} - --{{ $key }}={{ $value }} diff --git a/charts/kured/values.minikube.yaml b/charts/kured/values.minikube.yaml index cf4a554..dcc5297 100644 --- a/charts/kured/values.minikube.yaml +++ b/charts/kured/values.minikube.yaml @@ -9,6 +9,7 @@ configuration: # endTime: "" # only reboot before this time of day (default "23:59") # lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "1m" # reboot check period (default 1h0m0s) + # drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) # prometheusUrl: "" # Prometheus instance to probe for active alerts # rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) # rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -20,3 +21,5 @@ configuration: # startTime: "" # only reboot after this time of day (default "0:00") # timeZone: "" # time-zone to use (valid zones from "time" golang package) # annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations + # forceReboot: false # force a reboot even if the drain is still running (default false) + # forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index eb0022b..29f70c9 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -28,6 +28,7 @@ configuration: endTime: "" # only reboot before this time of day (default "23:59") lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") period: "" # reboot check period (default 1h0m0s) + drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1) prometheusUrl: "" # Prometheus instance to probe for active alerts rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -39,6 +40,8 @@ configuration: startTime: "" # only reboot after this time of day (default "0:00") timeZone: "" # time-zone to use (valid zones from "time" golang package) annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations + forceReboot: false # force a reboot even if the drain is still running (default false) + forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m) rbac: create: true diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 3810145..ddfe09f 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -38,7 +38,11 @@ var ( version = "unreleased" // Command line flags + forceDrain bool + forceReboot bool + forceTimeout time.Duration period time.Duration + drainGracePeriod int dsNamespace string dsName string lockAnnotation string @@ -91,6 +95,12 @@ func main() { PreRun: flagCheck, Run: root} + rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false, + "enable/disable force reboot") + rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1, + "drain grace period in seconds") + rootCmd.PersistentFlags().DurationVar(&forceTimeout, "force-timeout", time.Minute*30, + "total timeout which only applies when force-reboot is set to true") rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system", @@ -333,19 +343,27 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { drainer := &kubectldrain.Helper{ Client: client, - GracePeriodSeconds: -1, + Ctx: context.Background(), + GracePeriodSeconds: drainGracePeriod, Force: true, DeleteEmptyDirData: true, IgnoreAllDaemonSets: true, ErrOut: os.Stderr, Out: os.Stdout, } + if forceReboot { + drainer.Timeout = forceTimeout + } + if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { log.Fatalf("Error cordonning %s: %v", nodename, err) } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { - log.Fatalf("Error draining %s: %v", nodename, err) + if !forceReboot { + log.Fatalf("Error draining %s: %v", nodename, err) + } + log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err) } }