Implement universal notification mechanism

This patch gives the possibility to send notifications
 across different technologies. Also, this patch makes
 slack-hook-url, slack-username and slack-channel
 deprecated (informed by a warning).
 Also, updated the documentation (Readme).

README.md

@@ -219,21 +219,31 @@ If you choose to employ such an alert and have configured kured to
 probe for active alerts before rebooting, be sure to specify
 `--alert-filter-regexp=^RebootRequired$` to avoid deadlock!
 
-### Slack Notifications
-
-If you specify a Slack hook via `--slack-hook-url`, kured will notify
-you immediately prior to rebooting a node:
+### Notifications
 
+When you specify a formatted URL using `--notify-url`, kured will notify
+about draining and rebooting nodes across a list of technologies. 
 ![Notification](img/slack-notification.png)
 
-We recommend setting `--slack-username` to be the name of the
-environment, e.g. `dev` or `prod`.
-
 Alternatively you can use the `--message-template-drain` and `--message-template-reboot` to customize the text of the message, e.g.
 ```
 --message-template-drain="Draining node %s part of *my-cluster* in region *xyz*"
 ```
 
+Here is the syntax:
+
+slack:           `slack://tokenA/tokenB/tokenC`
+(`--slack-hook-url` is deprecated but possible to use)
+
+rocketchat:      `rocketchat://[username@]rocketchat-host/token[/channel|@recipient]`
+
+teams:           `teams://token-a/token-b/token-c`
+
+Email:           `smtp://username:password@host:port/?fromAddress=fromAddress&toAddresses=recipient1[,recipient2,...]`
+
+More details here: https://github.com/containrrr/shoutrrr/blob/master/docs/services/overview.md
+
+
 ### Overriding Lock Configuration
 
 The `--ds-name` and `--ds-namespace` arguments should match the name and

cmd/kured/main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/weaveworks/kured/pkg/notifications/slack"
 	"github.com/weaveworks/kured/pkg/taints"
 	"github.com/weaveworks/kured/pkg/timewindow"
+	shoutrrr "github.com/containrrr/shoutrrr"
 )
 
 var (
@@ -47,6 +48,7 @@ var (
 	alertFilter               *regexp.Regexp
 	rebootSentinelFile        string
 	rebootSentinelCommand     string
+	notifyURL                 string
 	slackHookURL              string
 	slackUsername             string
 	slackChannel              string
@@ -84,9 +86,10 @@ func init() {
 
 func main() {
 	rootCmd := &cobra.Command{
-		Use:   "kured",
-		Short: "Kubernetes Reboot Daemon",
-		Run:   root}
+		Use:    "kured",
+		Short:  "Kubernetes Reboot Daemon",
+		PreRun: flagCheck,
+		Run:    root}
 
 	rootCmd.PersistentFlags().DurationVar(&period, "period", time.Minute*60,
 		"sentinel check period")
@@ -116,7 +119,9 @@ func main() {
 	rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured",
 		"slack username for notifications")
 	rootCmd.PersistentFlags().StringVar(&slackChannel, "slack-channel", "",
-		"slack channel for notifications")
+		"slack channel for reboot notfications")
+	rootCmd.PersistentFlags().StringVar(&notifyURL, "notify-url", "",
+		"notify URL for reboot notfications")
 	rootCmd.PersistentFlags().StringVar(&messageTemplateDrain, "message-template-drain", "Draining node %s",
 		"message template used to notify about a node being drained")
 	rootCmd.PersistentFlags().StringVar(&messageTemplateReboot, "message-template-reboot", "Rebooting node %s",
@@ -142,7 +147,19 @@ func main() {
 	}
 }
 
-// newCommand wires a Command with stdout/stderr to our standard loggers
+// temporary func that checks for deprecated slack-notification-related flags
+func flagCheck(cmd *cobra.Command, args []string) {
+	if slackHookURL != "" && notifyURL != "" {
+		log.Warnf("Cannot use both --notify-url and --slack-hook-url flags. Kured will use --notify-url flag only...")
+		slackHookURL = ""
+	}
+	if slackChannel != "" || slackHookURL != "" || slackUsername != "" {
+		log.Warnf("Deprecated flag(s). Please use --notify-url flag instead.")
+	}
+
+}
+
+// newCommand creates a new Command with stdout/stderr wired to our standard logger
 func newCommand(name string, arg ...string) *exec.Cmd {
 	cmd := exec.Command(name, arg...)
 	cmd.Stdout = log.NewEntry(log.StandardLogger()).
@@ -308,6 +325,11 @@ func drain(client *kubernetes.Clientset, node *v1.Node) {
 			log.Warnf("Error notifying slack: %v", err)
 		}
 	}
+	if notifyURL != "" {
+		if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateDrain, nodename)); err != nil {
+			log.Warnf("Error notifying: %v", err)
+		}
+	}
 
 	drainer := &kubectldrain.Helper{
 		Client:              client,
@@ -349,8 +371,16 @@ func invokeReboot(nodeID string, rebootCommand []string) {
 		}
 	}
 
-	if err := newCommand(rebootCommand[0], rebootCommand[1:]...).Run(); err != nil {
-		log.Fatalf("Error invoking %s command: %v", rebootCommand, err)
+	if notifyURL != "" {
+		if err := shoutrrr.Send(notifyURL, fmt.Sprintf(messageTemplateReboot, nodeID)); err != nil {
+			log.Warnf("Error notifying: %v", err)
+		}
+	}
+
+	// Relies on hostPID:true and privileged:true to enter host mount space
+	rebootCmd := newCommand("/usr/bin/nsenter", "-m/proc/1/ns/mnt", "/bin/systemctl", "reboot")
+	if err := rebootCmd.Run(); err != nil {
+		log.Fatalf("Error invoking reboot command: %v", err)
 	}
 }
 

go.mod

@@ -3,7 +3,7 @@ module github.com/weaveworks/kured
 go 1.15
 
 require (
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/containrrr/shoutrrr v0.4.1
 	github.com/prometheus/client_golang v1.8.0
 	github.com/prometheus/common v0.15.0
 	github.com/sirupsen/logrus v1.8.1

go.sum

@@ -85,6 +85,7 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/containrrr/shoutrrr v0.4.1/go.mod h1:zqL2BvfC1W4FujrT4b3/ZCLxvD+uoeEpBL7rg9Dqpbg=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=