Merge pull request #14 from stefanprodan/slack

Add details to Slack messages

.codecov.yml

@@ -3,6 +3,6 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0.50
+        threshold: 50
         base: auto
-    patch: off
+    patch: off

README.md

@@ -367,7 +367,25 @@ helm upgrade -i flagger flagger/flagger \
 Once configured with a Slack incoming webhook, Flagger will post messages when a canary deployment has been initialized,
 when a new revision has been detected and if the canary analysis failed or succeeded.
 
-![flagger-slack](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-notifications.png)
+![flagger-slack](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-success.png)
+
+A canary deployment will be rolled back if the progress deadline exceeded or if the analysis 
+reached the maximum number of failed checks:
+
+![flagger-slack-errors](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/screens/slack-canary-failed.png)
+
+Besides Slack, you can use Alertmanager to trigger alerts when a canary deployment failed:
+
+```yaml
+  - alert: canary_rollback
+    expr: flagger_canary_status > 1
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Canary failed"
+      description: "Workload {{ $labels.name }} namespace {{ $labels.namespace }}"
+```
 
 ### Roadmap
 

pkg/controller/controller.go

@@ -260,12 +260,36 @@ func (c *Controller) recordEventWarningf(r *flaggerv1.Canary, template string, a
 	c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...))
 }
 
-func (c *Controller) sendNotification(workload string, namespace string, message string, warn bool) {
+func (c *Controller) sendNotification(cd *flaggerv1.Canary, message string, metadata bool, warn bool) {
 	if c.notifier == nil {
 		return
 	}
 
-	err := c.notifier.Post(workload, namespace, message, warn)
+	var fields []notifier.SlackField
+
+	if metadata {
+		fields = append(fields,
+			notifier.SlackField{
+				Title: "Target",
+				Value: fmt.Sprintf("%s/%s.%s", cd.Spec.TargetRef.Kind, cd.Spec.TargetRef.Name, cd.Namespace),
+			},
+			notifier.SlackField{
+				Title: "Traffic routing",
+				Value: fmt.Sprintf("Weight step: %v max: %v",
+					cd.Spec.CanaryAnalysis.StepWeight,
+					cd.Spec.CanaryAnalysis.MaxWeight),
+			},
+			notifier.SlackField{
+				Title: "Failed checks threshold",
+				Value: fmt.Sprintf("%v", cd.Spec.CanaryAnalysis.Threshold),
+			},
+			notifier.SlackField{
+				Title: "Progress deadline",
+				Value: fmt.Sprintf("%vs", cd.GetProgressDeadlineSeconds()),
+			},
+		)
+	}
+	err := c.notifier.Post(cd.Name, cd.Namespace, message, fields, warn)
 	if err != nil {
 		c.logger.Error(err)
 	}

pkg/controller/scheduler.go

@@ -95,11 +95,15 @@ func (c *Controller) advanceCanary(name string, namespace string) {
 		if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold {
 			c.recordEventWarningf(cd, "Rolling back %s.%s failed checks threshold reached %v",
 				cd.Name, cd.Namespace, cd.Status.FailedChecks)
+			c.sendNotification(cd, fmt.Sprintf("Failed checks threshold reached %v", cd.Status.FailedChecks),
+				false, true)
 		}
 
 		if !retriable {
 			c.recordEventWarningf(cd, "Rolling back %s.%s progress deadline exceeded %v",
 				cd.Name, cd.Namespace, err)
+			c.sendNotification(cd, fmt.Sprintf("Progress deadline exceeded %v", err),
+				false, true)
 		}
 
 		// route all traffic back to primary
@@ -112,7 +116,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
 
 		c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight)
 		c.recordEventWarningf(cd, "Canary failed! Scaling down %s.%s",
-			cd.Spec.TargetRef.Name, cd.Namespace)
+			cd.Name, cd.Namespace)
 
 		// shutdown canary
 		if err := c.deployer.Scale(cd, 0); err != nil {
@@ -127,8 +131,6 @@ func (c *Controller) advanceCanary(name string, namespace string) {
 		}
 
 		c.recorder.SetStatus(cd)
-		c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
-			"Canary analysis failed, rollback finished.", true)
 		return
 	}
 
@@ -199,8 +201,8 @@ func (c *Controller) advanceCanary(name string, namespace string) {
 			return
 		}
 		c.recorder.SetStatus(cd)
-		c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
-			"Canary analysis completed successfully, promotion finished.", false)
+		c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
+			false, false)
 	}
 }
 
@@ -217,15 +219,15 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl
 		}
 		c.recorder.SetStatus(cd)
 		c.recordEventInfof(cd, "Initialization done! %s.%s", cd.Name, cd.Namespace)
-		c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
-			"New deployment detected, initialization completed.", false)
+		c.sendNotification(cd, "New deployment detected, initialization completed.",
+			true, false)
 		return false
 	}
 
 	if diff, err := deployer.IsNewSpec(cd); diff {
 		c.recordEventInfof(cd, "New revision detected! Scaling up %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
-		c.sendNotification(cd.Spec.TargetRef.Name, cd.Namespace,
-			"New revision detected, starting canary analysis.", false)
+		c.sendNotification(cd, "New revision detected, starting canary analysis.",
+			true, false)
 		if err = deployer.Scale(cd, 1); err != nil {
 			c.recordEventErrorf(cd, "%v", err)
 			return false

pkg/notifier/slack.go

@@ -30,10 +30,17 @@ type SlackPayload struct {
 
 // SlackAttachment holds the markdown message body
 type SlackAttachment struct {
-	Color      string   `json:"color"`
-	AuthorName string   `json:"author_name"`
-	Text       string   `json:"text"`
-	MrkdwnIn   []string `json:"mrkdwn_in"`
+	Color      string       `json:"color"`
+	AuthorName string       `json:"author_name"`
+	Text       string       `json:"text"`
+	MrkdwnIn   []string     `json:"mrkdwn_in"`
+	Fields     []SlackField `json:"fields"`
+}
+
+type SlackField struct {
+	Title string `json:"title"`
+	Value string `json:"value"`
+	Short bool   `json:"short"`
 }
 
 // NewSlack validates the Slack URL and returns a Slack object
@@ -60,7 +67,7 @@ func NewSlack(hookURL string, username string, channel string) (*Slack, error) {
 }
 
 // Post Slack message
-func (s *Slack) Post(workload string, namespace string, message string, warn bool) error {
+func (s *Slack) Post(workload string, namespace string, message string, fields []SlackField, warn bool) error {
 	payload := SlackPayload{
 		Channel:  s.Channel,
 		Username: s.Username,
@@ -76,6 +83,7 @@ func (s *Slack) Post(workload string, namespace string, message string, warn boo
 		AuthorName: fmt.Sprintf("%s.%s", workload, namespace),
 		Text:       message,
 		MrkdwnIn:   []string{"text"},
+		Fields:     fields,
 	}
 
 	payload.Attachments = []SlackAttachment{a}