github/kured
1
0
Fork 0
mirror of https://github.com/kubereboot/kured.git synced 2026-08-23 21:36:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #212 from DaniJG/master

Browse Source 
GH-125, add additional parameters for the drain/reboot slack message template
This commit is contained in:
Daniel Holbach
2020-11-26 08:53:29 +01:00
committed by GitHub
parent 2fef8b1b12 51cab0dedc
commit 722db47b2d
10 changed files with 75 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DEVELOPMENT.md
+8
View File
@@ -47,8 +47,12 @@ sudo touch /var/run/reboot-required
A test-run with `minikube` could look like this:
```console
# start minikube
minikube start --vm-driver kvm2 --kubernetes-version <k8s-release>
# build kured image and publish to registry accessible by minikube
make image minikube-publish
# edit kured-ds.yaml to
# - point to new image
# - change e.g. period and reboot-days option for immediate results
@@ -57,6 +61,10 @@ minikube kubectl -- apply -f kured-rbac.yaml
minikube kubectl -- apply -f kured-ds.yaml
minikube kubectl -- logs daemonset.apps/kured -n kube-system -f
# Alternatively use helm to install the chart
# edit values-local.yaml to change any chart parameters
helm install kured ./charts/kured --namespace kube-system -f ./charts/kured/values.minikube.yaml
# In separate terminal
minikube ssh
sudo touch /var/run/reboot-required
README.md
+7
View File
@@ -94,6 +94,8 @@ Flags:
--slack-channel string slack channel for reboot notfications
--slack-hook-url string slack hook URL for reboot notfications
--slack-username string slack username for reboot notfications (default "kured")
--message-template-drain string message template used to notify about a node being drained (default "Draining node %s")
--message-template-reboot string message template used to notify about a node being rebooted (default "Rebooting node %s")
--start-time string only reboot after this time of day (default "0:00")
--time-zone string use this timezone to calculate allowed reboot time (default "UTC")
```
@@ -218,6 +220,11 @@ you immediately prior to rebooting a node:
We recommend setting `--slack-username` to be the name of the
environment, e.g. `dev` or `prod`.
Alternatively you can use the `--message-template-drain` and `--message-template-reboot` to customize the text of the message, e.g.
```
--message-template-drain="Draining node %s part of *my-cluster* in region *xyz*"
```
### Overriding Lock Configuration
The `--ds-name` and `--ds-namespace` arguments should match the name and
charts/kured/Chart.yaml
+1 -1
View File
@@ -2,7 +2,7 @@ apiVersion: v1
appVersion: "1.5.1"
description: A Helm chart for kured
name: kured
version: 2.2.1
version: 2.2.2
home: https://github.com/weaveworks/kured
maintainers:
- name: ckotzbauer
charts/kured/README.md
+3 -1
View File
@@ -55,6 +55,8 @@ The following changes have been made compared to the stable chart:
| `configuration.slackChannel` | cli-parameter `--slack-channel` | `""` |
| `configuration.slackHookUrl` | cli-parameter `--slack-hook-url` | `""` |
| `configuration.slackUsername` | cli-parameter `--slack-username` | `""` |
| `configuration.messageTemplateDrain` | cli-parameter `--message-template-drain` | `""` |
| `configuration.messageTemplateReboot` | cli-parameter `--message-template-reboot` | `""` |
| `configuration.startTime` | cli-parameter `--start-time` | `""` |
| `configuration.timeZone` | cli-parameter `--time-zone` | `""` |
| `rbac.create` | Create RBAC roles | `true` |
@@ -86,7 +88,7 @@ becomes `/usr/bin/kured ... --foo=1 --bar-baz=2`.
## Prometheus Metrics
Kured exposes a single prometheus metric indicating whether a reboot is required or not (see [kured docs](https://github.com/weaveworks/kured#prometheus-metrics)) for details.
Kured exposes a single prometheus metric indicating whether a reboot is required or not (see [kured docs](https://github.com/weaveworks/kured#prometheus-metrics)) for details.
#### Prometheus-Operator
charts/kured/templates/daemonset.yaml
+7 -1
View File
@@ -81,6 +81,12 @@ spec:
{{- if .Values.configuration.slackUsername }}
- --slack-username={{ .Values.configuration.slackUsername }}
{{- end }}
{{- if .Values.configuration.messageTemplateDrain }}
- --message-template-drain={{ .Values.configuration.messageTemplateDrain }}
{{- end }}
{{- if .Values.configuration.messageTemplateReboot }}
- --message-template-reboot={{ .Values.configuration.messageTemplateReboot }}
{{- end }}
{{- if .Values.configuration.startTime }}
- --start-time={{ .Values.configuration.startTime }}
{{- end }}
@@ -96,7 +102,7 @@ spec:
{{- end }}
ports:
- containerPort: 8080
name: metrics
name: metrics
env:
# Pass in the name of the node on which this pod is scheduled
# for use with drain/uncordon operations and lock acquisition
charts/kured/values.minikube.yaml
+21
View File
@@ -0,0 +1,21 @@
image:
repository: weaveworks/kured
tag: latest
configuration:
# annotationTtl: 0 # force clean annotation after this ammount of time (default 0, disabled)
# alertFilterRegexp: "" # alert names to ignore when checking for active alerts
# blockingPodSelector: [] # label selector identifying pods whose presence should prevent reboots
# endTime: "" # only reboot before this time of day (default "23:59")
# lockAnnotation: "" # annotation in which to record locking node (default "weave.works/kured-node-lock")
period: "1m" # reboot check period (default 1h0m0s)
# prometheusUrl: "" # Prometheus instance to probe for active alerts
# rebootDays: [] # only reboot on these days (default [su,mo,tu,we,th,fr,sa])
# rebootSentinel: "" # path to file whose existence signals need to reboot (default "/var/run/reboot-required")
# slackChannel: "" # slack channel for reboot notfications
# slackHookUrl: "" # slack hook URL for reboot notfications
# slackUsername: "" # slack username for reboot notfications (default "kured")
# messageTemplateDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s")
# messageTemplateReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s")
# startTime: "" # only reboot after this time of day (default "0:00")
# timeZone: "" # time-zone to use (valid zones from "time" golang package)
charts/kured/values.yaml
+2
View File
@@ -32,6 +32,8 @@ configuration:
slackChannel: "" # slack channel for reboot notfications
slackHookUrl: "" # slack hook URL for reboot notfications
slackUsername: "" # slack username for reboot notfications (default "kured")
messageTemplateDrain: "" # slack message template when notifying about a node being drained (default "Draining node %s")
messageTemplateReboot: "" # slack message template when notifying about a node being rebooted (default "Rebooted node %s")
startTime: "" # only reboot after this time of day (default "0:00")
timeZone: "" # time-zone to use (valid zones from "time" golang package)
cmd/kured/main.go
+20 -14
View File
@@ -31,18 +31,20 @@ var (
version = "unreleased"
// Command line flags
period time.Duration
dsNamespace string
dsName string
lockAnnotation string
lockTTL time.Duration
prometheusURL string
alertFilter *regexp.Regexp
rebootSentinel string
slackHookURL string
slackUsername string
slackChannel string
podSelectors []string
period time.Duration
dsNamespace string
dsName string
lockAnnotation string
lockTTL time.Duration
prometheusURL string
alertFilter *regexp.Regexp
rebootSentinel string
slackHookURL string
slackUsername string
slackChannel string
messageTemplateDrain string
messageTemplateReboot string
podSelectors []string
rebootDays []string
rebootStart string
@@ -90,6 +92,10 @@ func main() {
"slack username for reboot notfications")
rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "",
"slack channel for reboot notfications")
rootCmd.PersistentFlags().StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s",
"message template used to notify about a node being drained")
rootCmd.PersistentFlags().StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s",
"message template used to notify about a node being rebooted")
rootCmd.PersistentFlags().StringArrayVar(&podSelectors, "blocking-pod-selector", nil,
"label selector identifying pods whose presence should prevent reboots")
@@ -237,7 +243,7 @@ func drain(client *kubernetes.Clientset, node *v1.Node) {
log.Infof("Draining node %s", nodename)
if slackHookURL != "" {
if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, nodename); err != nil {
if err := slack.NotifyDrain(slackHookURL, slackUsername, slackChannel, messageTemplateDrain, nodename); err != nil {
log.Warnf("Error notifying slack: %v", err)
}
}
@@ -277,7 +283,7 @@ func commandReboot(nodeID string) {
log.Infof("Commanding reboot for node: %s", nodeID)
if slackHookURL != "" {
if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, nodeID); err != nil {
if err := slack.NotifyReboot(slackHookURL, slackUsername, slackChannel, messageTemplateReboot, nodeID); err != nil {
log.Warnf("Error notifying slack: %v", err)
}
}
kured-ds.yaml
+2
View File
@@ -59,5 +59,7 @@ spec:
# - --slack-hook-url=https://hooks.slack.com/...
# - --slack-username=prod
# - --slack-channel=alerting
# - --message-template-drain=Draining node %s
# - --message-template-drain=Rebooting node %s
# - --start-time=0:00
# - --time-zone=UTC
pkg/notifications/slack/slack.go
+4 -4
View File
@@ -44,11 +44,11 @@ func notify(hookURL, username, channel, message string) error {
}
// NotifyDrain is the exposed way to notify of a drain event onto a slack chan
func NotifyDrain(hookURL, username, channel, nodeID string) error {
return notify(hookURL, username, channel, fmt.Sprintf("Draining node %s", nodeID))
func NotifyDrain(hookURL, username, channel, messageTemplate, nodeID string) error {
return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID))
}
// NotifyReboot is the exposed way to notify of a reboot event onto a slack chan
func NotifyReboot(hookURL, username, channel, nodeID string) error {
return notify(hookURL, username, channel, fmt.Sprintf("Rebooting node %s", nodeID))
func NotifyReboot(hookURL, username, channel, messageTemplate, nodeID string) error {
return notify(hookURL, username, channel, fmt.Sprintf(messageTemplate, nodeID))
}