From 48897eb0abd6d2b8e3a74c41177f10f8d15345a8 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Sat, 28 Nov 2020 21:51:50 +0100 Subject: [PATCH 1/9] avoid indentations to ease readability --- cmd/kured/main.go | 72 +++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index fd876ea..f412a4c 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -31,20 +31,20 @@ var ( version = "unreleased" // Command line flags - period time.Duration - dsNamespace string - dsName string - lockAnnotation string - lockTTL time.Duration - prometheusURL string - alertFilter *regexp.Regexp - rebootSentinel string - slackHookURL string - slackUsername string - slackChannel string - messageTemplateDrain string - messageTemplateReboot string - podSelectors []string + period time.Duration + dsNamespace string + dsName string + lockAnnotation string + lockTTL time.Duration + prometheusURL string + alertFilter *regexp.Regexp + rebootSentinel string + slackHookURL string + slackUsername string + slackChannel string + messageTemplateDrain string + messageTemplateReboot string + podSelectors []string rebootDays []string rebootStart string @@ -339,23 +339,35 @@ func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Dur source := rand.NewSource(time.Now().UnixNano()) tick := delaytick.New(source, period) for range tick { - if window.Contains(time.Now()) && rebootRequired() && !rebootBlocked(client, nodeID) { - node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{}) - if err != nil { - log.Fatal(err) - } - nodeMeta.Unschedulable = node.Spec.Unschedulable + if !window.Contains(time.Now()) { + continue + } - if acquire(lock, &nodeMeta, TTL) { - if !nodeMeta.Unschedulable { - drain(client, node) - } - commandReboot(nodeID) - for { - log.Infof("Waiting for reboot") - time.Sleep(time.Minute) - } - } + if !rebootRequired() { + continue + } + + if rebootBlocked(client, nodeID) { + continue + } + + node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{}) + if err != nil { + log.Fatal(err) + } + nodeMeta.Unschedulable = node.Spec.Unschedulable + + if !acquire(lock, &nodeMeta, TTL) { + continue + } + + if !nodeMeta.Unschedulable { + drain(client, node) + } + commandReboot(nodeID) + for { + log.Infof("Waiting for reboot") + time.Sleep(time.Minute) } } } From 204a06ca38303a78025e5eb8b3907c3a25156188 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Sat, 28 Nov 2020 21:52:35 +0100 Subject: [PATCH 2/9] fixed call of log.Fatal instead of log.Fatalf --- cmd/kured/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index f412a4c..a931288 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -258,11 +258,11 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { Out: os.Stdout, } if err := kubectldrain.RunCordonOrUncordon(drainer, node, true); err != nil { - log.Fatal("Error cordonning %s: %v", nodename, err) + log.Fatalf("Error cordonning %s: %v", nodename, err) } if err := kubectldrain.RunNodeDrain(drainer, nodename); err != nil { - log.Fatal("Error draining %s: %v", nodename, err) + log.Fatalf("Error draining %s: %v", nodename, err) } } @@ -275,7 +275,7 @@ func uncordon(client *kubernetes.Clientset, node *v1.Node) { Out: os.Stdout, } if err := kubectldrain.RunCordonOrUncordon(drainer, node, false); err != nil { - log.Fatal("Error uncordonning %s: %v", nodename, err) + log.Fatalf("Error uncordonning %s: %v", nodename, err) } } From e4c684c3aff11890b64a944ea1f5dde0000a5046 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Sat, 28 Nov 2020 22:52:08 +0100 Subject: [PATCH 3/9] taint node with PreferNoSchedule to prevent receiving (and double draining) additional pods from other rebooting nodes --- cmd/kured/main.go | 44 +++++++++----- pkg/taints/taints.go | 140 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 14 deletions(-) create mode 100644 pkg/taints/taints.go diff --git a/cmd/kured/main.go b/cmd/kured/main.go index a931288..9e0a971 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -24,6 +24,7 @@ import ( "github.com/weaveworks/kured/pkg/daemonsetlock" "github.com/weaveworks/kured/pkg/delaytick" "github.com/weaveworks/kured/pkg/notifications/slack" + "github.com/weaveworks/kured/pkg/taints" "github.com/weaveworks/kured/pkg/timewindow" ) @@ -31,20 +32,21 @@ var ( version = "unreleased" // Command line flags - period time.Duration - dsNamespace string - dsName string - lockAnnotation string - lockTTL time.Duration - prometheusURL string - alertFilter *regexp.Regexp - rebootSentinel string - slackHookURL string - slackUsername string - slackChannel string - messageTemplateDrain string - messageTemplateReboot string - podSelectors []string + period time.Duration + dsNamespace string + dsName string + lockAnnotation string + lockTTL time.Duration + prometheusURL string + alertFilter *regexp.Regexp + rebootSentinel string + preferNoScheduleTaintName string + slackHookURL string + slackUsername string + slackChannel string + messageTemplateDrain string + messageTemplateReboot string + podSelectors []string rebootDays []string rebootStart string @@ -85,6 +87,8 @@ func main() { "alert names to ignore when checking for active alerts") rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required", "path to file whose existence signals need to reboot") + rootCmd.PersistentFlags().StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "weave.works/kured-node-reboot", + "taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes)") rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "", "slack hook URL for reboot notfications") @@ -336,10 +340,19 @@ func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Dur release(lock) } + preferNoScheduleTaint := taints.New(client, nodeID, preferNoScheduleTaintName, v1.TaintEffectPreferNoSchedule) + + // Remove taint immediately during startup to quickly allow scheduling again. + if !rebootRequired() { + preferNoScheduleTaint.Disable() + } + source := rand.NewSource(time.Now().UnixNano()) tick := delaytick.New(source, period) for range tick { if !window.Contains(time.Now()) { + // Remove taint outside the reboot time window to allow for normal operation. + preferNoScheduleTaint.Disable() continue } @@ -358,6 +371,8 @@ func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Dur nodeMeta.Unschedulable = node.Spec.Unschedulable if !acquire(lock, &nodeMeta, TTL) { + // Prefer to not schedule pods onto this node to avoid draing the same pod multiple times. + preferNoScheduleTaint.Enable() continue } @@ -392,6 +407,7 @@ func root(cmd *cobra.Command, args []string) { } else { log.Info("Lock TTL not set, lock will remain until being released") } + log.Infof("PreferNoSchedule taint: %s", preferNoScheduleTaintName) log.Infof("Reboot Sentinel: %s every %v", rebootSentinel, period) log.Infof("Blocking Pod Selectors: %v", podSelectors) log.Infof("Reboot on: %v", window) diff --git a/pkg/taints/taints.go b/pkg/taints/taints.go new file mode 100644 index 0000000..1e3a6a0 --- /dev/null +++ b/pkg/taints/taints.go @@ -0,0 +1,140 @@ +package taints + +import ( + "context" + "encoding/json" + "fmt" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +// Taint allows to set soft and hard limitations for scheduling and executing pods on nodes. +type Taint struct { + client *kubernetes.Clientset + nodeID string + taintName string + effect v1.TaintEffect +} + +// New provides a new taint. +func New(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect) *Taint { + return &Taint{ + client: client, + nodeID: nodeID, + taintName: taintName, + effect: effect, + } +} + +// Enable creates the taint for a node. Creating an existing taint is a noop. +func (t *Taint) Enable() { + preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, true) +} + +// Disable removes the taint for a node. Removing a missing taint is a noop. +func (t *Taint) Disable() { + preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, false) +} + +func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, taintShouldExists bool) { + updatedNode, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{}) + if err != nil || updatedNode == nil { + log.Fatalf("Error reading node %s: %v", nodeID, err) + } + + taintExists := false + offset := 0 + for i, taint := range updatedNode.Spec.Taints { + if taint.Key == taintName { + taintExists = true + offset = i + break + } + } + + if taintExists && taintShouldExists { + log.Debugf("Taint %v exists already for node %v.", taintName, nodeID) + return + } + + if !taintExists && !taintShouldExists { + log.Debugf("Taint %v already missing for node %v.", taintName, nodeID) + return + } + + type patchTaints struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` + } + + taint := v1.Taint{ + Key: taintName, + Effect: effect, + } + + var patches []patchTaints + + if len(updatedNode.Spec.Taints) == 0 { + // add first taint and ensure to keep current taints + patches = []patchTaints{ + { + Op: "test", + Path: "/spec", + Value: updatedNode.Spec, + }, + { + Op: "add", + Path: "/spec/taints", + Value: []v1.Taint{}, + }, + { + Op: "add", + Path: "/spec/taints/-", + Value: taint, + }, + } + } else if taintExists { + // remove taint and ensure to test against race conditions + patches = []patchTaints{ + { + Op: "test", + Path: fmt.Sprintf("/spec/taints/%d", offset), + Value: taint, + }, + { + Op: "remove", + Path: fmt.Sprintf("/spec/taints/%d", offset), + }, + } + } else { + // add missing taint to exsting list + patches = []patchTaints{ + { + Op: "add", + Path: "/spec/taints/-", + Value: taint, + }, + } + } + + patchBytes, err := json.Marshal(patches) + if err != nil { + log.Fatalf("Error encoding taint patcht for node %s: %v", nodeID, err) + } + + _, err = client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) + if err != nil { + log.Fatalf("Error patching taint for node %s: %v", nodeID, err) + } + + if taintShouldExists { + log.Info("Node taint added") + } else { + log.Info("Node taint removed") + } +} From 10d95c426f5ad243d5b67defc34b54ab744b5651 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Wed, 6 Jan 2021 21:29:35 +0100 Subject: [PATCH 4/9] fixed type & renamed variable --- pkg/taints/taints.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/taints/taints.go b/pkg/taints/taints.go index 1e3a6a0..56dd41e 100644 --- a/pkg/taints/taints.go +++ b/pkg/taints/taints.go @@ -40,7 +40,7 @@ func (t *Taint) Disable() { preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, false) } -func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, taintShouldExists bool) { +func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, shouldExists bool) { updatedNode, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{}) if err != nil || updatedNode == nil { log.Fatalf("Error reading node %s: %v", nodeID, err) @@ -56,12 +56,12 @@ func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, ef } } - if taintExists && taintShouldExists { + if taintExists && shouldExists { log.Debugf("Taint %v exists already for node %v.", taintName, nodeID) return } - if !taintExists && !taintShouldExists { + if !taintExists && !shouldExists { log.Debugf("Taint %v already missing for node %v.", taintName, nodeID) return } @@ -124,7 +124,7 @@ func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, ef patchBytes, err := json.Marshal(patches) if err != nil { - log.Fatalf("Error encoding taint patcht for node %s: %v", nodeID, err) + log.Fatalf("Error encoding taint patch for node %s: %v", nodeID, err) } _, err = client.CoreV1().Nodes().Patch(context.TODO(), nodeID, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) @@ -132,7 +132,7 @@ func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, ef log.Fatalf("Error patching taint for node %s: %v", nodeID, err) } - if taintShouldExists { + if shouldExists { log.Info("Node taint added") } else { log.Info("Node taint removed") From 34446f949edf0d8abec99417d7e0df683df20eae Mon Sep 17 00:00:00 2001 From: David Sauer Date: Wed, 6 Jan 2021 21:39:32 +0100 Subject: [PATCH 5/9] Allow to disable tainting during pending node reboot by setting the taint name to an empty string. --- cmd/kured/main.go | 2 +- pkg/taints/taints.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 9e0a971..7cea012 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -88,7 +88,7 @@ func main() { rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required", "path to file whose existence signals need to reboot") rootCmd.PersistentFlags().StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "weave.works/kured-node-reboot", - "taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes)") + "Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Set to \"\" to disable tainting.") rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "", "slack hook URL for reboot notfications") diff --git a/pkg/taints/taints.go b/pkg/taints/taints.go index 56dd41e..3796ade 100644 --- a/pkg/taints/taints.go +++ b/pkg/taints/taints.go @@ -32,11 +32,19 @@ func New(client *kubernetes.Clientset, nodeID, taintName string, effect v1.Taint // Enable creates the taint for a node. Creating an existing taint is a noop. func (t *Taint) Enable() { + if t.taintName == "" { + return + } + preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, true) } // Disable removes the taint for a node. Removing a missing taint is a noop. func (t *Taint) Disable() { + if t.taintName == "" { + return + } + preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, false) } From b3e39418babc26ea153d68728627ea6cca1e31a6 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Wed, 6 Jan 2021 21:51:43 +0100 Subject: [PATCH 6/9] cache taint state to avoid unnecessary API calls --- pkg/taints/taints.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/taints/taints.go b/pkg/taints/taints.go index 3796ade..a9592e9 100644 --- a/pkg/taints/taints.go +++ b/pkg/taints/taints.go @@ -18,15 +18,19 @@ type Taint struct { nodeID string taintName string effect v1.TaintEffect + exists bool } // New provides a new taint. func New(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect) *Taint { + exists, _, _ := taintExists(client, nodeID, taintName) + return &Taint{ client: client, nodeID: nodeID, taintName: taintName, effect: effect, + exists: exists, } } @@ -36,7 +40,13 @@ func (t *Taint) Enable() { return } + if t.exists { + return + } + preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, true) + + t.exists = true } // Disable removes the taint for a node. Removing a missing taint is a noop. @@ -45,25 +55,33 @@ func (t *Taint) Disable() { return } + if !t.exists { + return + } + preferNoSchedule(t.client, t.nodeID, t.taintName, t.effect, false) + + t.exists = false } -func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, shouldExists bool) { +func taintExists(client *kubernetes.Clientset, nodeID, taintName string) (bool, int, *v1.Node) { updatedNode, err := client.CoreV1().Nodes().Get(context.TODO(), nodeID, metav1.GetOptions{}) if err != nil || updatedNode == nil { log.Fatalf("Error reading node %s: %v", nodeID, err) } - taintExists := false - offset := 0 for i, taint := range updatedNode.Spec.Taints { if taint.Key == taintName { - taintExists = true - offset = i - break + return true, i, updatedNode } } + return false, 0, updatedNode +} + +func preferNoSchedule(client *kubernetes.Clientset, nodeID, taintName string, effect v1.TaintEffect, shouldExists bool) { + taintExists, offset, updatedNode := taintExists(client, nodeID, taintName) + if taintExists && shouldExists { log.Debugf("Taint %v exists already for node %v.", taintName, nodeID) return From e430b1442abdb4290e4f62a052b00aa2a7d81997 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Wed, 6 Jan 2021 21:58:22 +0100 Subject: [PATCH 7/9] updated README --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f36430a..a5c2063 100644 --- a/README.md +++ b/README.md @@ -79,25 +79,27 @@ The following arguments can be passed to kured via the daemonset pod template: ```console Flags: - --lock-ttl time force clean annotation after this ammount of time (default 0, disabled) --alert-filter-regexp regexp.Regexp alert names to ignore when checking for active alerts --blocking-pod-selector stringArray label selector identifying pods whose presence should prevent reboots --ds-name string name of daemonset on which to place lock (default "kured") --ds-namespace string namespace containing daemonset on which to place lock (default "kube-system") - --end-time string only reboot before this time of day (default "23:59") + --end-time string schedule reboot only before this time of day (default "23:59:59") -h, --help help for kured --lock-annotation string annotation in which to record locking node (default "weave.works/kured-node-lock") + --lock-ttl duration expire lock annotation after this duration (default: 0, disabled) + --message-template-drain string message template used to notify about a node being drained (default "Draining node %s") + --message-template-reboot string message template used to notify about a node being rebooted (default "Rebooting node %s") --period duration reboot check period (default 1h0m0s) + --prefer-no-schedule-taint string Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Set to "" to disable tainting. (default "weave.works/kured-node-reboot") --prometheus-url string Prometheus instance to probe for active alerts - --reboot-days strings only reboot on these days (default [su,mo,tu,we,th,fr,sa]) + --reboot-days strings schedule reboot on these days (default [su,mo,tu,we,th,fr,sa]) --reboot-sentinel string path to file whose existence signals need to reboot (default "/var/run/reboot-required") --slack-channel string slack channel for reboot notfications --slack-hook-url string slack hook URL for reboot notfications --slack-username string slack username for reboot notfications (default "kured") - --message-template-drain string message template used to notify about a node being drained (default "Draining node %s") - --message-template-reboot string message template used to notify about a node being rebooted (default "Rebooting node %s") - --start-time string only reboot after this time of day (default "0:00") - --time-zone string use this timezone to calculate allowed reboot time (default "UTC") + --start-time string schedule reboot only after this time of day (default "0:00") + --time-zone string use this timezone for schedule inputs (default "UTC") + ``` ### Reboot Sentinel File & Period From 3a35d6a46ce84ed75fc891157a7595c751f38eba Mon Sep 17 00:00:00 2001 From: David Sauer Date: Wed, 6 Jan 2021 22:21:41 +0100 Subject: [PATCH 8/9] remove taint in case the reboot is not needed anymore --- cmd/kured/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 7cea012..4056c7d 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -357,6 +357,7 @@ func rebootAsRequired(nodeID string, window *timewindow.TimeWindow, TTL time.Dur } if !rebootRequired() { + preferNoScheduleTaint.Disable() continue } From 5a4e197d27e894c0b8bcde7f033aacc4f5870719 Mon Sep 17 00:00:00 2001 From: David Sauer Date: Mon, 11 Jan 2021 18:21:23 +0100 Subject: [PATCH 9/9] change taint config to be disabled by default --- README.md | 3 +-- cmd/kured/main.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a5c2063..7b4ca4e 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Flags: --message-template-drain string message template used to notify about a node being drained (default "Draining node %s") --message-template-reboot string message template used to notify about a node being rebooted (default "Rebooting node %s") --period duration reboot check period (default 1h0m0s) - --prefer-no-schedule-taint string Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Set to "" to disable tainting. (default "weave.works/kured-node-reboot") + --prefer-no-schedule-taint string Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to "weave.works/kured-node-reboot" to enable tainting. --prometheus-url string Prometheus instance to probe for active alerts --reboot-days strings schedule reboot on these days (default [su,mo,tu,we,th,fr,sa]) --reboot-sentinel string path to file whose existence signals need to reboot (default "/var/run/reboot-required") @@ -99,7 +99,6 @@ Flags: --slack-username string slack username for reboot notfications (default "kured") --start-time string schedule reboot only after this time of day (default "0:00") --time-zone string use this timezone for schedule inputs (default "UTC") - ``` ### Reboot Sentinel File & Period diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 4056c7d..d2626fc 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -87,8 +87,8 @@ func main() { "alert names to ignore when checking for active alerts") rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required", "path to file whose existence signals need to reboot") - rootCmd.PersistentFlags().StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "weave.works/kured-node-reboot", - "Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Set to \"\" to disable tainting.") + rootCmd.PersistentFlags().StringVar(&preferNoScheduleTaintName, "prefer-no-schedule-taint", "", + "Taint name applied during pending node reboot (to prevent receiving additional pods from other rebooting nodes). Disabled by default. Set e.g. to \"weave.works/kured-node-reboot\" to enable tainting.") rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "", "slack hook URL for reboot notfications")