From f059cec794896f46888fd9e078c1f19f4c9cd69c Mon Sep 17 00:00:00 2001 From: Daniel Jimenez Garcia Date: Tue, 20 Oct 2020 21:00:01 +0100 Subject: [PATCH 1/2] GH-125, add additional parameters to override the drain/reboot slack messages --- DEVELOPMENT.md | 8 +++++++ README.md | 4 ++++ charts/kured/Chart.yaml | 2 +- charts/kured/README.md | 4 +++- charts/kured/templates/daemonset.yaml | 8 ++++++- charts/kured/values.minikube.yaml | 21 +++++++++++++++++ charts/kured/values.yaml | 2 ++ cmd/kured/main.go | 34 ++++++++++++++++----------- kured-ds.yaml | 2 ++ pkg/notifications/slack/slack.go | 8 +++---- 10 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 charts/kured/values.minikube.yaml diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 0d78589..8a0e975 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -47,8 +47,12 @@ sudo touch /var/run/reboot-required A test-run with `minikube` could look like this: ```console +# start minikube minikube start --vm-driver kvm2 --kubernetes-version +# build kured image and publish to registry accessible by minikube +make image minikube-publish + # edit kured-ds.yaml to # - point to new image # - change e.g. period and reboot-days option for immediate results @@ -57,6 +61,10 @@ minikube kubectl -- apply -f kured-rbac.yaml minikube kubectl -- apply -f kured-ds.yaml minikube kubectl -- logs daemonset.apps/kured -n kube-system -f +# Alternatively use helm to install the chart +# edit values-local.yaml to change any chart parameters +helm install kured ./charts/kured --namespace kube-system -f ./charts/kured/values.minikube.yaml + # In separate terminal minikube ssh sudo touch /var/run/reboot-required diff --git a/README.md b/README.md index e99418c..07911e4 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,8 @@ Flags: --slack-channel string slack channel for reboot notfications --slack-hook-url string slack hook URL for reboot notfications --slack-username string slack username for reboot notfications (default "kured") + --slack-message-drain string slack message template when notifying about a node being drained (default "Draining node %s") + --slack-message-reboot string slack message template when notifying about a node being rebooted (default "Rebooting node %s") --start-time string only reboot after this time of day (default "0:00") --time-zone string use this timezone to calculate allowed reboot time (default "UTC") ``` @@ -218,6 +220,8 @@ you immediately prior to rebooting a node: We recommend setting `--slack-username` to be the name of the environment, e.g. `dev` or `prod`. +Alternatively you can use the `--slack-message-reboot` and `--slack-message-reboot` to customize the text of the message, e.g. `"Draining node %s part of my-cluster in region xyz"` + ### Overriding Lock Configuration The `--ds-name` and `--ds-namespace` arguments should match the name and diff --git a/charts/kured/Chart.yaml b/charts/kured/Chart.yaml index c5ce2ec..794a453 100644 --- a/charts/kured/Chart.yaml +++ b/charts/kured/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: "1.5.1" description: A Helm chart for kured name: kured -version: 2.2.1 +version: 2.2.2 home: https://github.com/weaveworks/kured maintainers: - name: ckotzbauer diff --git a/charts/kured/README.md b/charts/kured/README.md index 440dd50..2655394 100644 --- a/charts/kured/README.md +++ b/charts/kured/README.md @@ -55,6 +55,8 @@ The following changes have been made compared to the stable chart: | `configuration.slackChannel` | cli-parameter `--slack-channel` | `""` | | `configuration.slackHookUrl` | cli-parameter `--slack-hook-url` | `""` | | `configuration.slackUsername` | cli-parameter `--slack-username` | `""` | +| `configuration.slackMessageDrain` | cli-parameter `--slack-message-drain` | `""` | +| `configuration.slackMessageReboot` | cli-parameter `--slack-message-reboot` | `""` | | `configuration.startTime` | cli-parameter `--start-time` | `""` | | `configuration.timeZone` | cli-parameter `--time-zone` | `""` | | `rbac.create` | Create RBAC roles | `true` | @@ -86,7 +88,7 @@ becomes `/usr/bin/kured ... --foo=1 --bar-baz=2`. ## Prometheus Metrics -Kured exposes a single prometheus metric indicating whether a reboot is required or not (see [kured docs](https://github.com/weaveworks/kured#prometheus-metrics)) for details. +Kured exposes a single prometheus metric indicating whether a reboot is required or not (see [kured docs](https://github.com/weaveworks/kured#prometheus-metrics)) for details. #### Prometheus-Operator diff --git a/charts/kured/templates/daemonset.yaml b/charts/kured/templates/daemonset.yaml index 83b0bd9..ffd8083 100644 --- a/charts/kured/templates/daemonset.yaml +++ b/charts/kured/templates/daemonset.yaml @@ -81,6 +81,12 @@ spec: {{- if .Values.configuration.slackUsername }} - --slack-username={{ .Values.configuration.slackUsername }} {{- end }} + {{- if .Values.configuration.slackMessageDrain }} + - --slack-message-drain={{ .Values.configuration.slackMessageDrain }} + {{- end }} + {{- if .Values.configuration.slackMessageReboot }} + - --slack-message-reboot={{ .Values.configuration.slackMessageReboot }} + {{- end }} {{- if .Values.configuration.startTime }} - --start-time={{ .Values.configuration.startTime }} {{- end }} @@ -96,7 +102,7 @@ spec: {{- end }} ports: - containerPort: 8080 - name: metrics + name: metrics env: # Pass in the name of the node on which this pod is scheduled # for use with drain/uncordon operations and lock acquisition diff --git a/charts/kured/values.minikube.yaml b/charts/kured/values.minikube.yaml new file mode 100644 index 0000000..0f3ab85 --- /dev/null +++ b/charts/kured/values.minikube.yaml @@ -0,0 +1,21 @@ +image: + repository: weaveworks/kured + tag: latest + +configuration: + # annotationTtl: 0 # force clean annotation after this ammount of time (default 0, disabled) + # alertFilterRegexp: "" # alert names to ignore when checking for active alerts + # blockingPodSelector: [] # label selector identifying pods whose presence should prevent reboots + # endTime: "" # only reboot before this time of day (default "23:59") + # lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock") + period: "1m" # reboot check period (default 1h0m0s) + # prometheusUrl: "" # Prometheus instance to probe for active alerts + # rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa]) + # rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required") + # slackChannel: "" # slack channel for reboot notfications + # slackHookUrl: "" # slack hook URL for reboot notfications + # slackUsername: "" # slack username for reboot notfications (default "kured") + # slackMessageDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s") + # slackMessageReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s") + # startTime: "" # only reboot after this time of day (default "0:00") + # timeZone: "" # time-zone to use (valid zones from "time" golang package) diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index 8de29f5..4533b37 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -32,6 +32,8 @@ configuration: slackChannel: "" # slack channel for reboot notfications slackHookUrl: "" # slack hook URL for reboot notfications slackUsername: "" # slack username for reboot notfications (default "kured") + slackMessageDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s") + slackMessageReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s") startTime: "" # only reboot after this time of day (default "0:00") timeZone: "" # time-zone to use (valid zones from "time" golang package) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 2da707c..6f8977c 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -31,18 +31,20 @@ var ( version = "unreleased" // Command line flags - period time.Duration - dsNamespace string - dsName string - lockAnnotation string - lockTTL time.Duration - prometheusURL string - alertFilter *regexp.Regexp - rebootSentinel string - slackHookURL string - slackUsername string - slackChannel string - podSelectors []string + period time.Duration + dsNamespace string + dsName string + lockAnnotation string + lockTTL time.Duration + prometheusURL string + alertFilter *regexp.Regexp + rebootSentinel string + slackHookURL string + slackUsername string + slackChannel string + slackMessageDrain string + slackMessageReboot string + podSelectors []string rebootDays []string rebootStart string @@ -90,6 +92,10 @@ func main() { "slack username for reboot notfications") rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "", "slack channel for reboot notfications") + rootCmd.PersistentFlags().StringVar(&slackMessageDrain, "slack-message-drain", "Draining node %s", + "slack message template when notifying about a node being drained") + rootCmd.PersistentFlags().StringVar(&slackMessageReboot, "slack-message-reboot", "Rebooting node %s", + "slack message template when notifying about a node being rebooted") rootCmd.PersistentFlags().StringArrayVar(&podSelectors, "blocking-pod-selector", nil, "label selector identifying pods whose presence should prevent reboots") @@ -237,7 +243,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { log.Infof("Draining node %s", nodename) if slackHookURL != "" { - if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, nodename); err != nil { + if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, slackMessageDrain, nodename); err != nil { log.Warnf("Error notifying slack: %v", err) } } @@ -277,7 +283,7 @@ func commandReboot(nodeID string) { log.Infof("Commanding reboot for node: %s", nodeID) if slackHookURL != "" { - if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, nodeID); err != nil { + if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, slackMessageReboot, nodeID); err != nil { log.Warnf("Error notifying slack: %v", err) } } diff --git a/kured-ds.yaml b/kured-ds.yaml index b2df4db..6cfa443 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml @@ -59,5 +59,7 @@ spec: # - --slack-hook-url=https://hooks.slack.com/... # - --slack-username=prod # - --slack-channel=alerting +# - --slack-message-drain=Draining node %s +# - --slack-message-drain=Rebooting node %s # - --start-time=0:00 # - --time-zone=UTC diff --git a/pkg/notifications/slack/slack.go b/pkg/notifications/slack/slack.go index 3ebb7ec..4af05d9 100644 --- a/pkg/notifications/slack/slack.go +++ b/pkg/notifications/slack/slack.go @@ -44,11 +44,11 @@ func notify(hookURL, username, channel, message string) error { } // NotifyDrain is the exposed way to notify of a drain event onto a slack chan -func NotifyDrain(hookURL, username, channel, nodeID string) error { - return notify(hookURL, username, channel, fmt.Sprintf("Draining node %s", nodeID)) +func NotifyDrain(hookURL, username, channel, messageTemplate, nodeID string) error { + return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID)) } // NotifyReboot is the exposed way to notify of a reboot event onto a slack chan -func NotifyReboot(hookURL, username, channel, nodeID string) error { - return notify(hookURL, username, channel, fmt.Sprintf("Rebooting node %s", nodeID)) +func NotifyReboot(hookURL, username, channel, messageTemplate, nodeID string) error { + return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID)) } From 51cab0dedc117b4fe7871cdb2afdaee31b58989b Mon Sep 17 00:00:00 2001 From: Daniel Jimenez Garcia Date: Tue, 17 Nov 2020 17:28:52 +0000 Subject: [PATCH 2/2] rename message template parameters so they are not related to slack --- README.md | 9 ++++-- charts/kured/README.md | 4 +-- charts/kured/templates/daemonset.yaml | 8 +++--- charts/kured/values.minikube.yaml | 4 +-- charts/kured/values.yaml | 4 +-- cmd/kured/main.go | 40 +++++++++++++-------------- kured-ds.yaml | 4 +-- 7 files changed, 38 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 07911e4..f8fce1c 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,8 @@ Flags: --slack-channel string slack channel for reboot notfications --slack-hook-url string slack hook URL for reboot notfications --slack-username string slack username for reboot notfications (default "kured") - --slack-message-drain string slack message template when notifying about a node being drained (default "Draining node %s") - --slack-message-reboot string slack message template when notifying about a node being rebooted (default "Rebooting node %s") + --message-template-drain string message template used to notify about a node being drained (default "Draining node %s") + --message-template-reboot string message template used to notify about a node being rebooted (default "Rebooting node %s") --start-time string only reboot after this time of day (default "0:00") --time-zone string use this timezone to calculate allowed reboot time (default "UTC") ``` @@ -220,7 +220,10 @@ you immediately prior to rebooting a node: We recommend setting `--slack-username` to be the name of the environment, e.g. `dev` or `prod`. -Alternatively you can use the `--slack-message-reboot` and `--slack-message-reboot` to customize the text of the message, e.g. `"Draining node %s part of my-cluster in region xyz"` +Alternatively you can use the `--message-template-drain` and `--message-template-reboot` to customize the text of the message, e.g. +``` +--message-template-drain="Draining node %s part of *my-cluster* in region *xyz*" +``` ### Overriding Lock Configuration diff --git a/charts/kured/README.md b/charts/kured/README.md index 2655394..7facd67 100644 --- a/charts/kured/README.md +++ b/charts/kured/README.md @@ -55,8 +55,8 @@ The following changes have been made compared to the stable chart: | `configuration.slackChannel` | cli-parameter `--slack-channel` | `""` | | `configuration.slackHookUrl` | cli-parameter `--slack-hook-url` | `""` | | `configuration.slackUsername` | cli-parameter `--slack-username` | `""` | -| `configuration.slackMessageDrain` | cli-parameter `--slack-message-drain` | `""` | -| `configuration.slackMessageReboot` | cli-parameter `--slack-message-reboot` | `""` | +| `configuration.messageTemplateDrain` | cli-parameter `--message-template-drain` | `""` | +| `configuration.messageTemplateReboot` | cli-parameter `--message-template-reboot` | `""` | | `configuration.startTime` | cli-parameter `--start-time` | `""` | | `configuration.timeZone` | cli-parameter `--time-zone` | `""` | | `rbac.create` | Create RBAC roles | `true` | diff --git a/charts/kured/templates/daemonset.yaml b/charts/kured/templates/daemonset.yaml index ffd8083..f6ba661 100644 --- a/charts/kured/templates/daemonset.yaml +++ b/charts/kured/templates/daemonset.yaml @@ -81,11 +81,11 @@ spec: {{- if .Values.configuration.slackUsername }} - --slack-username={{ .Values.configuration.slackUsername }} {{- end }} - {{- if .Values.configuration.slackMessageDrain }} - - --slack-message-drain={{ .Values.configuration.slackMessageDrain }} + {{- if .Values.configuration.messageTemplateDrain }} + - --message-template-drain={{ .Values.configuration.messageTemplateDrain }} {{- end }} - {{- if .Values.configuration.slackMessageReboot }} - - --slack-message-reboot={{ .Values.configuration.slackMessageReboot }} + {{- if .Values.configuration.messageTemplateReboot }} + - --message-template-reboot={{ .Values.configuration.messageTemplateReboot }} {{- end }} {{- if .Values.configuration.startTime }} - --start-time={{ .Values.configuration.startTime }} diff --git a/charts/kured/values.minikube.yaml b/charts/kured/values.minikube.yaml index 0f3ab85..9b27635 100644 --- a/charts/kured/values.minikube.yaml +++ b/charts/kured/values.minikube.yaml @@ -15,7 +15,7 @@ configuration: # slackChannel: "" # slack channel for reboot notfications # slackHookUrl: "" # slack hook URL for reboot notfications # slackUsername: "" # slack username for reboot notfications (default "kured") - # slackMessageDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s") - # slackMessageReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s") + # messageTemplateDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s") + # messageTemplateReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s") # startTime: "" # only reboot after this time of day (default "0:00") # timeZone: "" # time-zone to use (valid zones from "time" golang package) diff --git a/charts/kured/values.yaml b/charts/kured/values.yaml index 4533b37..f999e14 100644 --- a/charts/kured/values.yaml +++ b/charts/kured/values.yaml @@ -32,8 +32,8 @@ configuration: slackChannel: "" # slack channel for reboot notfications slackHookUrl: "" # slack hook URL for reboot notfications slackUsername: "" # slack username for reboot notfications (default "kured") - slackMessageDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s") - slackMessageReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s") + messageTemplateDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s") + messageTemplateReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s") startTime: "" # only reboot after this time of day (default "0:00") timeZone: "" # time-zone to use (valid zones from "time" golang package) diff --git a/cmd/kured/main.go b/cmd/kured/main.go index 6f8977c..fd876ea 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -31,20 +31,20 @@ var ( version = "unreleased" // Command line flags - period time.Duration - dsNamespace string - dsName string - lockAnnotation string - lockTTL time.Duration - prometheusURL string - alertFilter *regexp.Regexp - rebootSentinel string - slackHookURL string - slackUsername string - slackChannel string - slackMessageDrain string - slackMessageReboot string - podSelectors []string + period time.Duration + dsNamespace string + dsName string + lockAnnotation string + lockTTL time.Duration + prometheusURL string + alertFilter *regexp.Regexp + rebootSentinel string + slackHookURL string + slackUsername string + slackChannel string + messageTemplateDrain string + messageTemplateReboot string + podSelectors []string rebootDays []string rebootStart string @@ -92,10 +92,10 @@ func main() { "slack username for reboot notfications") rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "", "slack channel for reboot notfications") - rootCmd.PersistentFlags().StringVar(&slackMessageDrain, "slack-message-drain", "Draining node %s", - "slack message template when notifying about a node being drained") - rootCmd.PersistentFlags().StringVar(&slackMessageReboot, "slack-message-reboot", "Rebooting node %s", - "slack message template when notifying about a node being rebooted") + rootCmd.PersistentFlags().StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s", + "message template used to notify about a node being drained") + rootCmd.PersistentFlags().StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s", + "message template used to notify about a node being rebooted") rootCmd.PersistentFlags().StringArrayVar(&podSelectors, "blocking-pod-selector", nil, "label selector identifying pods whose presence should prevent reboots") @@ -243,7 +243,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { log.Infof("Draining node %s", nodename) if slackHookURL != "" { - if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, slackMessageDrain, nodename); err != nil { + if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, messageTemplateDrain, nodename); err != nil { log.Warnf("Error notifying slack: %v", err) } } @@ -283,7 +283,7 @@ func commandReboot(nodeID string) { log.Infof("Commanding reboot for node: %s", nodeID) if slackHookURL != "" { - if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, slackMessageReboot, nodeID); err != nil { + if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, messageTemplateReboot, nodeID); err != nil { log.Warnf("Error notifying slack: %v", err) } } diff --git a/kured-ds.yaml b/kured-ds.yaml index 6cfa443..2e0ffae 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml @@ -59,7 +59,7 @@ spec: # - --slack-hook-url=https://hooks.slack.com/... # - --slack-username=prod # - --slack-channel=alerting -# - --slack-message-drain=Draining node %s -# - --slack-message-drain=Rebooting node %s +# - --message-template-drain=Draining node %s +# - --message-template-drain=Rebooting node %s # - --start-time=0:00 # - --time-zone=UTC