Reloader Reporting and Alerting #320

Alert a webhook supporting service ( e.g, slack ) whenever reloader reloads
workload on change

internal/pkg/alerts/alert.go

@@ -0,0 +1,94 @@
+package alert
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/parnurzeal/gorequest"
+	"github.com/sirupsen/logrus"
+)
+
+// function to send alert msg to webhook service
+func SendWebhookAlert(msg string) {
+	webhook_url, ok := os.LookupEnv("ALERT_WEBHOOK_URL")
+	if !ok {
+		logrus.Error("ALERT_WEBHOOK_URL env variable not provided")
+		return
+	}
+	webhook_url = strings.TrimSpace(webhook_url)
+	alert_sink := os.Getenv("ALERT_SINK")
+	alert_sink = strings.ToLower(strings.TrimSpace(alert_sink))
+
+	// Provision to add Proxy to reach webhook server if required
+	webhook_proxy := os.Getenv("ALERT_WEBHOOK_PROXY")
+	webhook_proxy = strings.TrimSpace(webhook_proxy)
+
+	// Provision to add Additional information in the alert. e.g ClusterName
+	alert_additional_info, ok := os.LookupEnv("ALERT_ADDITIONAL_INFO")
+	if ok {
+		alert_additional_info = strings.TrimSpace(alert_additional_info)
+		msg = fmt.Sprintf("%s : %s", alert_additional_info, msg)
+	}
+
+	if alert_sink == "slack" {
+		sendSlackAlert(webhook_url, webhook_proxy, msg)
+	} else {
+		msg = strings.Replace(msg, "*", "", -1)
+		sendRawWebhookAlert(webhook_url, webhook_proxy, msg)
+	}
+}
+
+// function to handle server redirection
+func redirectPolicy(req gorequest.Request, via []gorequest.Request) error {
+	return fmt.Errorf("incorrect token (redirection)")
+}
+
+// function to send alert to slack
+func sendSlackAlert(webhookUrl string, proxy string, msg string) []error {
+	attachment := Attachment{
+		Text:       msg,
+		Color:      "good",
+		AuthorName: "Reloader",
+	}
+
+	payload := WebhookMessage{
+		Attachments: []Attachment{attachment},
+	}
+
+	request := gorequest.New().Proxy(proxy)
+	resp, _, err := request.
+		Post(webhookUrl).
+		RedirectPolicy(redirectPolicy).
+		Send(payload).
+		End()
+
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		return []error{fmt.Errorf("error sending msg. status: %v", resp.Status)}
+	}
+
+	return nil
+}
+
+// function to send alert to webhook service as text
+func sendRawWebhookAlert(webhookUrl string, proxy string, msg string) []error {
+	request := gorequest.New().Proxy(proxy)
+	resp, _, err := request.
+		Post(webhookUrl).
+		Type("text").
+		RedirectPolicy(redirectPolicy).
+		Send(msg).
+		End()
+
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		return []error{fmt.Errorf("error sending msg. status: %v", resp.Status)}
+	}
+
+	return nil
+}

internal/pkg/alerts/slack_alert.go

@@ -0,0 +1,61 @@
+package alert
+
+type WebhookMessage struct {
+	Username        string       `json:"username,omitempty"`
+	IconEmoji       string       `json:"icon_emoji,omitempty"`
+	IconURL         string       `json:"icon_url,omitempty"`
+	Channel         string       `json:"channel,omitempty"`
+	ThreadTimestamp string       `json:"thread_ts,omitempty"`
+	Text            string       `json:"text,omitempty"`
+	Attachments     []Attachment `json:"attachments,omitempty"`
+	Parse           string       `json:"parse,omitempty"`
+	ResponseType    string       `json:"response_type,omitempty"`
+	ReplaceOriginal bool         `json:"replace_original,omitempty"`
+	DeleteOriginal  bool         `json:"delete_original,omitempty"`
+	ReplyBroadcast  bool         `json:"reply_broadcast,omitempty"`
+}
+
+type Attachment struct {
+	Color    string `json:"color,omitempty"`
+	Fallback string `json:"fallback,omitempty"`
+
+	CallbackID string `json:"callback_id,omitempty"`
+	ID         int    `json:"id,omitempty"`
+
+	AuthorID      string `json:"author_id,omitempty"`
+	AuthorName    string `json:"author_name,omitempty"`
+	AuthorSubname string `json:"author_subname,omitempty"`
+	AuthorLink    string `json:"author_link,omitempty"`
+	AuthorIcon    string `json:"author_icon,omitempty"`
+
+	Title     string `json:"title,omitempty"`
+	TitleLink string `json:"title_link,omitempty"`
+	Pretext   string `json:"pretext,omitempty"`
+	Text      string `json:"text,omitempty"`
+
+	ImageURL string `json:"image_url,omitempty"`
+	ThumbURL string `json:"thumb_url,omitempty"`
+
+	ServiceName string `json:"service_name,omitempty"`
+	ServiceIcon string `json:"service_icon,omitempty"`
+	FromURL     string `json:"from_url,omitempty"`
+	OriginalURL string `json:"original_url,omitempty"`
+
+	MarkdownIn []string `json:"mrkdwn_in,omitempty"`
+
+	Footer     string `json:"footer,omitempty"`
+	FooterIcon string `json:"footer_icon,omitempty"`
+}
+
+type Field struct {
+	Title string `json:"title"`
+	Value string `json:"value"`
+	Short bool   `json:"short"`
+}
+
+type Action struct {
+	Type  string `json:"type"`
+	Text  string `json:"text"`
+	Url   string `json:"url"`
+	Style string `json:"style"`
+}

internal/pkg/handler/upgrade.go

@@ -4,8 +4,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+	alert "github.com/stakater/Reloader/internal/pkg/alerts"
 	"github.com/stakater/Reloader/internal/pkg/callbacks"
 	"github.com/stakater/Reloader/internal/pkg/constants"
 	"github.com/stakater/Reloader/internal/pkg/metrics"
@@ -184,6 +187,13 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				logrus.Infof("Changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace)
 				logrus.Infof("Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace)
 				collectors.Reloaded.With(prometheus.Labels{"success": "true"}).Inc()
+				alert_on_reload, ok := os.LookupEnv("ALERT_ON_RELOAD")
+				if ok && alert_on_reload == "true" {
+					msg := fmt.Sprintf(
+						"Reloader detected changes in *%s* of type *%s* in namespace *%s*. Hence reloaded *%s* of type *%s* in namespace *%s*",
+						config.ResourceName, config.Type, config.Namespace, resourceName, upgradeFuncs.ResourceType, config.Namespace)
+					alert.SendWebhookAlert(msg)
+				}
 			}
 		}
 	}