From 1a6e814a21473beeaf82ac1b4e80be24dc935fd0 Mon Sep 17 00:00:00 2001 From: Adam Harrison Date: Tue, 15 Aug 2017 17:54:32 +0100 Subject: [PATCH] Basic slack notification support --- cmd/kured/Dockerfile | 1 + cmd/kured/main.go | 19 +++++++++++++-- kured-ds.yaml | 1 + pkg/notifications/slack/slack.go | 42 ++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 pkg/notifications/slack/slack.go diff --git a/cmd/kured/Dockerfile b/cmd/kured/Dockerfile index 27c4340..a2d3565 100644 --- a/cmd/kured/Dockerfile +++ b/cmd/kured/Dockerfile @@ -1,4 +1,5 @@ FROM ubuntu +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/cache/apt ADD https://storage.googleapis.com/kubernetes-release/release/v1.4.8/bin/linux/amd64/kubectl /usr/bin/kubectl RUN chmod 0755 /usr/bin/kubectl COPY ./kured /usr/bin/kured diff --git a/cmd/kured/main.go b/cmd/kured/main.go index e836a88..fb245e9 100644 --- a/cmd/kured/main.go +++ b/cmd/kured/main.go @@ -16,6 +16,7 @@ import ( "github.com/weaveworks/kured/pkg/alerts" "github.com/weaveworks/kured/pkg/daemonsetlock" "github.com/weaveworks/kured/pkg/delaytick" + "github.com/weaveworks/kured/pkg/notifications/slack" ) var ( @@ -27,6 +28,8 @@ var ( prometheusURL string alertFilter *regexp.Regexp rebootSentinel string + slackHookURL string + slackUsername string ) func main() { @@ -50,6 +53,11 @@ func main() { rootCmd.PersistentFlags().StringVar(&rebootSentinel, "reboot-sentinel", "/var/run/reboot-required", "path to file whose existence signals need to reboot") + rootCmd.PersistentFlags().StringVar(&slackHookURL, "slack-hook-url", "", + "slack hook URL for reboot notfications") + rootCmd.PersistentFlags().StringVar(&slackUsername, "slack-username", "kured", + "slack username for reboot notfications") + if err := rootCmd.Execute(); err != nil { log.Fatal(err) } @@ -169,8 +177,15 @@ func waitForDrain(client *kubernetes.Clientset, nodeID string) { } } -func reboot() { +func reboot(nodeID string) { log.Infof("Commanding reboot") + + if slackHookURL != "" { + if err := slack.NotifyReboot(slackHookURL, slackUsername, nodeID); err != nil { + log.Warnf("Error notifying slack: %v", err) + } + } + // Relies on /var/run/dbus/system_bus_socket bind mount to talk to systemd rebootCmd := exec.Command("/bin/systemctl", "reboot") if err := rebootCmd.Run(); err != nil { @@ -237,7 +252,7 @@ func root(cmd *cobra.Command, args []string) { drain(nodeID) waitForDrain(client, nodeID) } - reboot() + reboot(nodeID) break } } diff --git a/kured-ds.yaml b/kured-ds.yaml index a1ef249..e2a9895 100644 --- a/kured-ds.yaml +++ b/kured-ds.yaml @@ -23,6 +23,7 @@ spec: # - --period=60 # - --prometheus-url=http://prometheus.monitoring.svc.cluster.local # - --reboot-sentinel=/var/run/reboot-required +# - --slack-hook-url=https://hooks.slack.com/... env: # Pass in the name of the node on which this pod is scheduled # for use with drain/uncordon operations and lock acquisition diff --git a/pkg/notifications/slack/slack.go b/pkg/notifications/slack/slack.go new file mode 100644 index 0000000..0595daf --- /dev/null +++ b/pkg/notifications/slack/slack.go @@ -0,0 +1,42 @@ +package slack + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" +) + +var ( + httpClient = &http.Client{Timeout: 5 * time.Second} +) + +type body struct { + Text string `json:"text,omitempty"` + Username string `json:"username,omitempty"` +} + +func NotifyReboot(hookURL, username, nodeID string) error { + msg := body{ + Text: fmt.Sprintf("Rebooting node %s", nodeID), + Username: username, + } + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(&msg); err != nil { + return err + } + + resp, err := httpClient.Post(hookURL, "application/json", &buf) + defer resp.Body.Close() + if err != nil { + return err + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf(resp.Status) + } + + return nil +}