diff --git a/README.md b/README.md index 19d4f3a..446cf4f 100644 --- a/README.md +++ b/README.md @@ -219,21 +219,31 @@ If you choose to employ such an alert and have configured kured to probe for active alerts before rebooting, be sure to specify `--alert-filter-regexp=^RebootRequired$` to avoid deadlock! -### Slack Notifications - -If you specify a Slack hook via `--slack-hook-url`, kured will notify -you immediately prior to rebooting a node: +### Notifications +When you specify a formatted URL using `--notify-url`, kured will notify +about draining and rebooting nodes across a list of technologies. ![Notification](img/slack-notification.png) -We recommend setting `--slack-username` to be the name of the -environment, e.g. `dev` or `prod`. - Alternatively you can use the `--message-template-drain` and `--message-template-reboot` to customize the text of the message, e.g. ``` --message-template-drain="Draining node %s part of *my-cluster* in region *xyz*" ``` +Here is the syntax: + +slack: `slack://tokenA/tokenB/tokenC` +(`--slack-hook-url` is deprecated but possible to use) + +rocketchat: `rocketchat://[username@]rocketchat-host/token[/channel|@recipient]` + +teams: `teams://token-a/token-b/token-c` + +Email: `smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]` + +More details here: https://github.com/containrrr/shoutrrr/blob/master/docs/services/overview.md + + ### Overriding Lock Configuration The `--ds-name` and `--ds-namespace` arguments should match the name and diff --git a/cmd/kured/main.go b/cmd/kured/main.go index d495dec..b8dea93 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -31,6 +31,7 @@ import ( "github.com/weaveworks/kured/pkg/notifications/slack" "github.com/weaveworks/kured/pkg/taints" "github.com/weaveworks/kured/pkg/timewindow" + shoutrrr "github.com/containrrr/shoutrrr" ) var ( @@ -47,6 +48,7 @@ var ( alertFilter *regexp.Regexp rebootSentinelFile string rebootSentinelCommand string + notifyURL string slackHookURL string slackUsername string slackChannel string @@ -84,9 +86,10 @@ func init() { func main() { rootCmd := &cobra.Command{ - Use: "kured", - Short: "Kubernetes Reboot Daemon", - Run: root} + Use: "kured", + Short: "Kubernetes Reboot Daemon", + PreRun: flagCheck, + Run: root} rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60, "sentinel check period") @@ -116,7 +119,9 @@ func main() { rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured", "slack username for notifications") rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "", - "slack channel for notifications") + "slack channel for reboot notfications") + rootCmd.PersistentFlags().StringVar(¬ifyURL, "notify-url", "", + "notify URL for reboot notfications") rootCmd.PersistentFlags().StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s", "message template used to notify about a node being drained") rootCmd.PersistentFlags().StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s", @@ -142,7 +147,19 @@ func main() { } } -// newCommand wires a Command with stdout/stderr to our standard loggers +// temporary func that checks for deprecated slack-notification-related flags +func flagCheck(cmd *cobra.Command, args []string) { + if slackHookURL != "" && notifyURL != "" { + log.Warnf("Cannot use both --notify-url and --slack-hook-url flags. Kured will use --notify-url flag only...") + slackHookURL = "" + } + if slackChannel != "" || slackHookURL != "" || slackUsername != "" { + log.Warnf("Deprecated flag(s). Please use --notify-url flag instead.") + } + +} + +// newCommand creates a new Command with stdout/stderr wired to our standard logger func newCommand(name string, arg ...string) *exec.Cmd { cmd := exec.Command(name, arg...) cmd.Stdout = log.NewEntry(log.StandardLogger()). @@ -308,6 +325,11 @@ func drain(client *kubernetes.Clientset, node *v1.Node) { log.Warnf("Error notifying slack: %v", err) } } + if notifyURL != "" { + if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateDrain, nodename)); err != nil { + log.Warnf("Error notifying: %v", err) + } + } drainer := &kubectldrain.Helper{ Client: client, @@ -349,8 +371,16 @@ func invokeReboot(nodeID string, rebootCommand []string) { } } - if err := newCommand(rebootCommand[0], rebootCommand[1:]...).Run(); err != nil { - log.Fatalf("Error invoking %s command: %v", rebootCommand, err) + if notifyURL != "" { + if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateReboot, nodeID)); err != nil { + log.Warnf("Error notifying: %v", err) + } + } + + // Relies on hostPID:true and privileged:true to enter host mount space + rebootCmd := newCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/bin/systemctl", "reboot") + if err := rebootCmd.Run(); err != nil { + log.Fatalf("Error invoking reboot command: %v", err) } } diff --git a/go.mod b/go.mod index bfaeb8e..06cb4bd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/weaveworks/kured go 1.15 require ( - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/containrrr/shoutrrr v0.4.1 github.com/prometheus/client_golang v1.8.0 github.com/prometheus/common v0.15.0 github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index 43a0ae7..c6ea4aa 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,7 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containrrr/shoutrrr v0.4.1/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=