Reloader Reporting and Alerting #320

Alert a webhook supporting service ( e.g, slack ) whenever reloader reloads
workload on change

deployments/kubernetes/chart/reloader/templates/secret.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.reloader.deployment.env.secret -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "reloader-fullname" . }}
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+data:
+  {{ if .Values.reloader.deployment.env.secret.ALERT_ON_RELOAD -}}
+  ALERT_ON_RELOAD: {{ .Values.reloader.deployment.env.secret.ALERT_ON_RELOAD | b64enc | quote }}
+  {{ end }}
+  {{- if .Values.reloader.deployment.env.secret.ALERT_SINK -}}
+  ALERT_SINK: {{ .Values.reloader.deployment.env.secret.ALERT_SINK | b64enc | quote }}
+  {{ end }}
+  {{- if .Values.reloader.deployment.env.secret.ALERT_WEBHOOK_URL -}}
+  ALERT_WEBHOOK_URL: {{ .Values.reloader.deployment.env.secret.ALERT_WEBHOOK_URL | b64enc | quote }}
+  {{ end }}
+  {{- if .Values.reloader.deployment.env.secret.ALERT_ADDITIONAL_INFO -}}
+  ALERT_ADDITIONAL_INFO: {{ .Values.reloader.deployment.env.secret.ALERT_ADDITIONAL_INFO | b64enc | quote }}
+  {{ end }}
+{{ end }}

deployments/kubernetes/chart/reloader/values.yaml

@@ -73,6 +73,10 @@ reloader:
       open:
       # secret supports Key value pair as environment variables. It gets the values based on keys from default reloader secret if any.
       secret:
+      #  ALERT_ON_RELOAD: <"true"|"false">
+      #  ALERT_SINK: <"slack"> # By default it will be a raw text based webhook
+      #  ALERT_WEBHOOK_URL: <"webhook_url">
+      #  ALERT_ADDITIONAL_INFO: <"Additional Info like Cluster Name if needed">
       # field supports Key value pair as environment variables. It gets the values from other fields of pod.
       field:
 

go.mod

@@ -15,6 +15,7 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.58.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -42,6 +43,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/onsi/ginkgo v1.15.1 // indirect
 	github.com/onsi/gomega v1.11.0 // indirect
+	github.com/parnurzeal/gorequest v0.2.16 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.21.0 // indirect
@@ -61,6 +63,7 @@ require (
 	k8s.io/klog/v2 v2.8.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210216185858-15cd8face8d6 // indirect
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	moul.io/http2curl v1.0.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.0.2 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )

go.sum

@@ -18,6 +18,7 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP
 cloud.google.com/go v0.55.0/go.mod h1:ZHmoY+/lIMNkN2+fBmuTiqZ4inFhvQad8ft7MT8IV5Y=
 cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
 cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.58.0 h1:vtAfVc723K3xKq1BQydk/FyCldnaNFhGhpJxaJzgRMQ=
 cloud.google.com/go v0.58.0/go.mod h1:W+9FnSUw6nhVwXlFcp1eL+krq5+HQUJeUogSeJZZiWg=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
@@ -811,6 +812,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
+github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ=
+github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
@@ -1568,6 +1571,8 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
+moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=
 mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w=
 mvdan.cc/gofumpt v0.0.0-20200802201014-ab5a8192947d/go.mod h1:bzrjFmaD6+xqohD3KYP0H2FEuxknnBmyyOxdhLdaIws=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=

internal/pkg/alerts/alert.go

@@ -0,0 +1,94 @@
+package alert
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/parnurzeal/gorequest"
+	"github.com/sirupsen/logrus"
+)
+
+// function to send alert msg to webhook service
+func SendWebhookAlert(msg string) {
+	webhook_url, ok := os.LookupEnv("ALERT_WEBHOOK_URL")
+	if !ok {
+		logrus.Error("ALERT_WEBHOOK_URL env variable not provided")
+		return
+	}
+	webhook_url = strings.TrimSpace(webhook_url)
+	alert_sink := os.Getenv("ALERT_SINK")
+	alert_sink = strings.ToLower(strings.TrimSpace(alert_sink))
+
+	// Provision to add Proxy to reach webhook server if required
+	webhook_proxy := os.Getenv("ALERT_WEBHOOK_PROXY")
+	webhook_proxy = strings.TrimSpace(webhook_proxy)
+
+	// Provision to add Additional information in the alert. e.g ClusterName
+	alert_additional_info, ok := os.LookupEnv("ALERT_ADDITIONAL_INFO")
+	if ok {
+		alert_additional_info = strings.TrimSpace(alert_additional_info)
+		msg = fmt.Sprintf("%s : %s", alert_additional_info, msg)
+	}
+
+	if alert_sink == "slack" {
+		sendSlackAlert(webhook_url, webhook_proxy, msg)
+	} else {
+		msg = strings.Replace(msg, "*", "", -1)
+		sendRawWebhookAlert(webhook_url, webhook_proxy, msg)
+	}
+}
+
+// function to handle server redirection
+func redirectPolicy(req gorequest.Request, via []gorequest.Request) error {
+	return fmt.Errorf("incorrect token (redirection)")
+}
+
+// function to send alert to slack
+func sendSlackAlert(webhookUrl string, proxy string, msg string) []error {
+	attachment := Attachment{
+		Text:       msg,
+		Color:      "good",
+		AuthorName: "Reloader",
+	}
+
+	payload := WebhookMessage{
+		Attachments: []Attachment{attachment},
+	}
+
+	request := gorequest.New().Proxy(proxy)
+	resp, _, err := request.
+		Post(webhookUrl).
+		RedirectPolicy(redirectPolicy).
+		Send(payload).
+		End()
+
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		return []error{fmt.Errorf("error sending msg. status: %v", resp.Status)}
+	}
+
+	return nil
+}
+
+// function to send alert to webhook service as text
+func sendRawWebhookAlert(webhookUrl string, proxy string, msg string) []error {
+	request := gorequest.New().Proxy(proxy)
+	resp, _, err := request.
+		Post(webhookUrl).
+		Type("text").
+		RedirectPolicy(redirectPolicy).
+		Send(msg).
+		End()
+
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 400 {
+		return []error{fmt.Errorf("error sending msg. status: %v", resp.Status)}
+	}
+
+	return nil
+}

internal/pkg/alerts/slack_alert.go

@@ -0,0 +1,61 @@
+package alert
+
+type WebhookMessage struct {
+	Username        string       `json:"username,omitempty"`
+	IconEmoji       string       `json:"icon_emoji,omitempty"`
+	IconURL         string       `json:"icon_url,omitempty"`
+	Channel         string       `json:"channel,omitempty"`
+	ThreadTimestamp string       `json:"thread_ts,omitempty"`
+	Text            string       `json:"text,omitempty"`
+	Attachments     []Attachment `json:"attachments,omitempty"`
+	Parse           string       `json:"parse,omitempty"`
+	ResponseType    string       `json:"response_type,omitempty"`
+	ReplaceOriginal bool         `json:"replace_original,omitempty"`
+	DeleteOriginal  bool         `json:"delete_original,omitempty"`
+	ReplyBroadcast  bool         `json:"reply_broadcast,omitempty"`
+}
+
+type Attachment struct {
+	Color    string `json:"color,omitempty"`
+	Fallback string `json:"fallback,omitempty"`
+
+	CallbackID string `json:"callback_id,omitempty"`
+	ID         int    `json:"id,omitempty"`
+
+	AuthorID      string `json:"author_id,omitempty"`
+	AuthorName    string `json:"author_name,omitempty"`
+	AuthorSubname string `json:"author_subname,omitempty"`
+	AuthorLink    string `json:"author_link,omitempty"`
+	AuthorIcon    string `json:"author_icon,omitempty"`
+
+	Title     string `json:"title,omitempty"`
+	TitleLink string `json:"title_link,omitempty"`
+	Pretext   string `json:"pretext,omitempty"`
+	Text      string `json:"text,omitempty"`
+
+	ImageURL string `json:"image_url,omitempty"`
+	ThumbURL string `json:"thumb_url,omitempty"`
+
+	ServiceName string `json:"service_name,omitempty"`
+	ServiceIcon string `json:"service_icon,omitempty"`
+	FromURL     string `json:"from_url,omitempty"`
+	OriginalURL string `json:"original_url,omitempty"`
+
+	MarkdownIn []string `json:"mrkdwn_in,omitempty"`
+
+	Footer     string `json:"footer,omitempty"`
+	FooterIcon string `json:"footer_icon,omitempty"`
+}
+
+type Field struct {
+	Title string `json:"title"`
+	Value string `json:"value"`
+	Short bool   `json:"short"`
+}
+
+type Action struct {
+	Type  string `json:"type"`
+	Text  string `json:"text"`
+	Url   string `json:"url"`
+	Style string `json:"style"`
+}

internal/pkg/handler/upgrade.go

@@ -4,8 +4,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+	alert "github.com/stakater/Reloader/internal/pkg/alerts"
 	"github.com/stakater/Reloader/internal/pkg/callbacks"
 	"github.com/stakater/Reloader/internal/pkg/constants"
 	"github.com/stakater/Reloader/internal/pkg/metrics"
@@ -184,6 +187,13 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				logrus.Infof("Changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace)
 				logrus.Infof("Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace)
 				collectors.Reloaded.With(prometheus.Labels{"success": "true"}).Inc()
+				alert_on_reload, ok := os.LookupEnv("ALERT_ON_RELOAD")
+				if ok && alert_on_reload == "true" {
+					msg := fmt.Sprintf(
+						"Reloader detected changes in *%s* of type *%s* in namespace *%s*. Hence reloaded *%s* of type *%s* in namespace *%s*",
+						config.ResourceName, config.Type, config.Namespace, resourceName, upgradeFuncs.ResourceType, config.Namespace)
+					alert.SendWebhookAlert(msg)
+				}
 			}
 		}
 	}