github/kured
1
0
Fork 0
mirror of https://github.com/kubereboot/kured.git synced 2026-08-28 05:47:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add force-reboot after force-timeout duration has been exceeded

Browse Source
This commit is contained in:
Cameron McAvoy
2021-04-07 09:39:01 -05:00
parent d2c9ef8cba
commit 65292983f2
7 changed files with 42 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3
View File
@@ -83,9 +83,12 @@ The following arguments can be passed to kured via the daemonset pod template:
Flags:
--alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts
--blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots
--drain-grace-period int period of time to wait for node drain in seconds (default -1)
--ds-name string name of daemonset on which to place lock (default "kured")
--ds-namespace string namespace containing daemonset on which to place lock (default "kube-system")
--end-time string schedule reboot only before this time of day (default "23:59:59")
--force-reboot bool force a reboot even if the drain is still running (default false)
--force-timeout duration total drain timeout, only applies when force-reboot is enabled (default 30m)
-h, --help help for kured
--lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock")
--lock-ttl duration expire lock annotation after this duration (default: 0, disabled)
charts/kured/Chart.yaml
+1 -1
View File
@@ -2,7 +2,7 @@ apiVersion: v1
appVersion: "1.6.1"
description: A Helm chart for kured
name: kured
version: 2.4.2
version: 2.4.3
home: https://github.com/weaveworks/kured
maintainers:
- name: ckotzbauer
charts/kured/README.md
+3
View File
@@ -50,6 +50,7 @@ The following changes have been made compared to the stable chart:
| `configuration.endTime` | cli-parameter `--end-time` | `""` |
| `configuration.lockAnnotation` | cli-parameter `--lock-annotation` | `""` |
| `configuration.period` | cli-parameter `--period` | `""` |
| `configuration.drainGracePeriod` | cli-parameter `--drain-grace-period` | `""` |
| `configuration.prometheusUrl` | cli-parameter `--prometheus-url` | `""` |
| `configuration.rebootDays` | Array of days for multiple cli-parameters `--reboot-days` | `[]` |
| `configuration.rebootSentinel` | cli-parameter `--reboot-sentinel` | `""` |
@@ -61,6 +62,8 @@ The following changes have been made compared to the stable chart:
| `configuration.startTime` | cli-parameter `--start-time` | `""` |
| `configuration.timeZone` | cli-parameter `--time-zone` | `""` |
| `configuration.annotateNodes` | cli-parameter `--annotate-nodes` | `false` |
| `configuration.forceReboot` | cli-parameter `--force-reboot` | `false` |
| `configuration.forceTimeout` | cli-parameter `--force-timeout` | `""` |
| `rbac.create` | Create RBAC roles | `true` |
| `serviceAccount.create` | Create a service account | `true` |
| `serviceAccount.name` | Service account name to create (or use if `serviceAccount.create` is false) | (chart fullname) |
charts/kured/templates/daemonset.yaml
+9
View File
@@ -70,6 +70,9 @@ spec:
{{- if .Values.configuration.period }}
- --period={{ .Values.configuration.period }}
{{- end }}
{{- if .Values.configuration.drainGracePeriod }}
- --drain-grace-period={{ .Values.configuration.drainGracePeriod }}
{{- end }}
{{- if .Values.configuration.prometheusUrl }}
- --prometheus-url={{ .Values.configuration.prometheusUrl }}
{{- end }}
@@ -103,6 +106,12 @@ spec:
{{- if .Values.configuration.annotateNodes }}
- --annotate-nodes={{ .Values.configuration.annotateNodes }}
{{- end }}
{{- if .Values.configuration.forceReboot }}
- --force-reboot
{{- end }}
{{- if .Values.configuration.forceTimeout }}
- --force-timeout={{ .Values.configuration.forceTimeout }}
{{- end }}
{{- range $key, $value := .Values.extraArgs }}
{{- if $value }}
- --{{ $key }}={{ $value }}
charts/kured/values.minikube.yaml
+3
View File
@@ -9,6 +9,7 @@ configuration:
# endTime: "" # only reboot before this time of day (default "23:59")
# lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock")
period: "1m" # reboot check period (default 1h0m0s)
# drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1)
# prometheusUrl: "" # Prometheus instance to probe for active alerts
# rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa])
# rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required")
@@ -20,3 +21,5 @@ configuration:
# startTime: "" # only reboot after this time of day (default "0:00")
# timeZone: "" # time-zone to use (valid zones from "time" golang package)
# annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations
# forceReboot: false # force a reboot even if the drain is still running (default false)
# forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m)
charts/kured/values.yaml
+3
View File
@@ -28,6 +28,7 @@ configuration:
endTime: "" # only reboot before this time of day (default "23:59")
lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock")
period: "" # reboot check period (default 1h0m0s)
drainGracePeriod: "" # period of time to wait for node drain in seconds (default -1)
prometheusUrl: "" # Prometheus instance to probe for active alerts
rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa])
rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required")
@@ -39,6 +40,8 @@ configuration:
startTime: "" # only reboot after this time of day (default "0:00")
timeZone: "" # time-zone to use (valid zones from "time" golang package)
annotateNodes: false # enable 'weave.works/kured-reboot-in-progress' and 'weave.works/kured-most-recent-reboot-needed' node annotations to signify kured reboot operations
forceReboot: false # force a reboot even if the drain is still running (default false)
forceTimeout: "" # total drain timeout, only applies when force-reboot is enabled (default 30m)
rbac:
create: true
cmd/kured/main.go
+20 -2
View File
@@ -38,7 +38,11 @@ var (
version = "unreleased"
// Command line flags
forceDrain bool
forceReboot bool
forceTimeout time.Duration
period time.Duration
drainGracePeriod int
dsNamespace string
dsName string
lockAnnotation string
@@ -91,6 +95,12 @@ func main() {
PreRun: flagCheck,
Run: root}
rootCmd.PersistentFlags().BoolVar(&forceReboot, "force-reboot", false,
"enable/disable force reboot")
rootCmd.PersistentFlags().IntVar(&drainGracePeriod, "drain-grace-period", -1,
"drain grace period in seconds")
rootCmd.PersistentFlags().DurationVar(&forceTimeout, "force-timeout", time.Minute*30,
"total timeout which only applies when force-reboot is set to true")
rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60,
"sentinel check period")
rootCmd.PersistentFlags().StringVar(&dsNamespace, "ds-namespace", "kube-system",
@@ -333,19 +343,27 @@ func drain(client *kubernetes.Clientset, node *v1.Node) {
drainer := &kubectldrain.Helper{
Client: client,
GracePeriodSeconds: -1,
Ctx: context.Background(),
GracePeriodSeconds: drainGracePeriod,
Force: true,
DeleteEmptyDirData: true,
IgnoreAllDaemonSets: true,
ErrOut: os.Stderr,
Out: os.Stdout,
}
if forceReboot {
drainer.Timeout = forceTimeout
}
if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil {
log.Fatalf("Error cordonning %s: %v", nodename, err)
}
if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil {
log.Fatalf("Error draining %s: %v", nodename, err)
if !forceReboot {
log.Fatalf("Error draining %s: %v", nodename, err)
}
log.Errorf("Error draining %s: %v, continuing with reboot anyway", nodename, err)
}
}