Merge pull request #235 from weaveworks/msteams

Implement MS Teams notifications
This commit is contained in:
Stefan Prodan
2019-07-07 11:40:00 +03:00
committed by GitHub
15 changed files with 268 additions and 54 deletions
+15 -3
View File
@@ -19,13 +19,24 @@ Add Flagger Helm repository:
helm repo add flagger https://flagger.app
```
To install the chart with the release name `flagger`:
To install the chart with the release name `flagger` for Istio:
```console
$ helm install --name flagger --namespace istio-system flagger/flagger
$ helm upgrade -i flagger flagger/flagger \
--namespace=istio-system \
--set meshProvider=istio \
--set metricsServer=http://prometheus:9090
```
To install the chart with the release name `flagger` for Linkerd:
```console
$ helm upgrade -i flagger flagger/flagger \
--namespace=linkerd \
--set meshProvider=linkerd \
--set metricsServer=http://linkerd-prometheus:9090
```
The command deploys Flagger on the Kubernetes cluster in the istio-system namespace.
The [configuration](#configuration) section lists the parameters that can be configured during installation.
## Uninstalling the Chart
@@ -52,6 +63,7 @@ Parameter | Description | Default
`slack.url` | Slack incoming webhook | None
`slack.channel` | Slack channel | None
`slack.user` | Slack username | `flagger`
`msteams.url` | Microsoft Teams incoming webhook | None
`rbac.create` | if `true`, create and use RBAC resources | `true`
`rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `false`
`crd.create` | if `true`, create Flagger's CRDs | `true`
+3
View File
@@ -51,6 +51,9 @@ spec:
- -slack-user={{ .Values.slack.user }}
- -slack-channel={{ .Values.slack.channel }}
{{- end }}
{{- if .Values.msteams.url }}
- -msteams-url={{ .Values.msteams.url }}
{{- end }}
livenessProbe:
exec:
command:
+4
View File
@@ -19,6 +19,10 @@ slack:
# incoming webhook https://api.slack.com/incoming-webhooks
url:
msteams:
# MS Teams incoming webhook URL
url:
serviceAccount:
# serviceAccount.create: Whether to create a service account or not
create: true
+26 -11
View File
@@ -34,6 +34,7 @@ var (
controlLoopInterval time.Duration
logLevel string
port string
msteamsURL string
slackURL string
slackUser string
slackChannel string
@@ -56,6 +57,7 @@ func init() {
flag.StringVar(&slackURL, "slack-url", "", "Slack hook URL.")
flag.StringVar(&slackUser, "slack-user", "flagger", "Slack user name.")
flag.StringVar(&slackChannel, "slack-channel", "", "Slack channel.")
flag.StringVar(&msteamsURL, "msteams-url", "", "MS Teams incoming webhook URL.")
flag.IntVar(&threadiness, "threadiness", 2, "Worker concurrency.")
flag.BoolVar(&zapReplaceGlobals, "zap-replace-globals", false, "Whether to change the logging level of the global zap logger.")
flag.StringVar(&zapEncoding, "zap-encoding", "json", "Zap logger encoding.")
@@ -158,15 +160,8 @@ func main() {
logger.Errorf("Metrics server %s unreachable %v", metricsServer, err)
}
var slack *notifier.Slack
if slackURL != "" {
slack, err = notifier.NewSlack(slackURL, slackUser, slackChannel)
if err != nil {
logger.Errorf("Notifier %v", err)
} else {
logger.Infof("Slack notifications enabled for channel %s", slack.Channel)
}
}
// setup Slack or MS Teams notifications
notifierClient := initNotifier(logger)
// start HTTP server
go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
@@ -179,9 +174,8 @@ func main() {
flaggerClient,
canaryInformer,
controlLoopInterval,
metricsServer,
logger,
slack,
notifierClient,
routerFactory,
observerFactory,
meshProvider,
@@ -209,3 +203,24 @@ func main() {
<-stopCh
}
func initNotifier(logger *zap.SugaredLogger) (client notifier.Interface) {
provider := "slack"
notifierURL := slackURL
if msteamsURL != "" {
provider = "msteams"
notifierURL = msteamsURL
}
notifierFactory := notifier.NewFactory(notifierURL, slackUser, slackChannel)
if notifierURL != "" {
var err error
client, err = notifierFactory.Notifier(provider)
if err != nil {
logger.Errorf("Notifier %v", err)
} else {
logger.Infof("Notifications enabled for %s", notifierURL[0:30])
}
}
return
}
+2 -2
View File
@@ -5,12 +5,12 @@ description: Flagger is a progressive delivery Kubernetes operator
# Introduction
[Flagger](https://github.com/weaveworks/flagger) is a **Kubernetes** operator that automates the promotion of canary
deployments using **Istio**, **App Mesh**, **NGINX** or **Gloo** routing for traffic shifting and **Prometheus** metrics for canary analysis.
deployments using **Istio**, **Linkerd**, **App Mesh**, **NGINX** or **Gloo** routing for traffic shifting and **Prometheus** metrics for canary analysis.
The canary analysis can be extended with webhooks for running system integration/acceptance tests, load tests, or any other custom validation.
Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance
indicators like HTTP requests success rate, requests average duration and pods health.
Based on analysis of the **KPIs** a canary is promoted or aborted, and the analysis result is published to **Slack**.
Based on analysis of the **KPIs** a canary is promoted or aborted, and the analysis result is published to **Slack** or **MS Teams**.
![Flagger overview diagram](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-overview.png)
@@ -47,6 +47,14 @@ helm upgrade -i flagger flagger/flagger \
--set slack.user=flagger
```
Enable **Microsoft Teams** notifications:
```bash
helm upgrade -i flagger flagger/flagger \
--namespace=istio-system \
--set msteams.url=https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK
```
If you don't have Tiller you can use the helm template command and apply the generated yaml with kubectl:
```bash
+17 -1
View File
@@ -6,7 +6,6 @@ Flagger can be configured to send Slack notifications:
```bash
helm upgrade -i flagger flagger/flagger \
--namespace=istio-system \
--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
--set slack.channel=general \
--set slack.user=flagger
@@ -22,6 +21,23 @@ maximum number of failed checks:
![Slack Notifications](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/slack-canary-failed.png)
### Microsoft Teams
Flagger can be configured to send notifications to Microsoft Teams:
```bash
helm upgrade -i flagger flagger/flagger \
--set msteams.url=https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK
```
Flagger will post a message card to MS Teams when a new revision has been detected and if the canary analysis failed or succeeded:
![MS Teams Notifications](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/flagger-ms-teams-notifications.png)
And you'll get a notification on rollback:
![MS Teams Notifications](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/screens/flagger-ms-teams-failed.png)
### Prometheus Alert Manager
Besides Slack, you can use Alertmanager to trigger alerts when a canary deployment failed:
Binary file not shown.

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

+28 -16
View File
@@ -46,7 +46,7 @@ type Controller struct {
jobs map[string]CanaryJob
deployer canary.Deployer
recorder metrics.Recorder
notifier *notifier.Slack
notifier notifier.Interface
routerFactory *router.Factory
observerFactory *metrics.Factory
meshProvider string
@@ -58,9 +58,8 @@ func NewController(
flaggerClient clientset.Interface,
flaggerInformer flaggerinformers.CanaryInformer,
flaggerWindow time.Duration,
metricServer string,
logger *zap.SugaredLogger,
notifier *notifier.Slack,
notifier notifier.Interface,
routerFactory *router.Factory,
observerFactory *metrics.Factory,
meshProvider string,
@@ -271,29 +270,42 @@ func (c *Controller) sendNotification(cd *flaggerv1.Canary, message string, meta
return
}
var fields []notifier.SlackField
var fields []notifier.Field
if metadata {
fields = append(fields,
notifier.SlackField{
Title: "Target",
notifier.Field{
Name: "Target",
Value: fmt.Sprintf("%s/%s.%s", cd.Spec.TargetRef.Kind, cd.Spec.TargetRef.Name, cd.Namespace),
},
notifier.SlackField{
Title: "Traffic routing",
Value: fmt.Sprintf("Weight step: %v max: %v",
cd.Spec.CanaryAnalysis.StepWeight,
cd.Spec.CanaryAnalysis.MaxWeight),
},
notifier.SlackField{
Title: "Failed checks threshold",
notifier.Field{
Name: "Failed checks threshold",
Value: fmt.Sprintf("%v", cd.Spec.CanaryAnalysis.Threshold),
},
notifier.SlackField{
Title: "Progress deadline",
notifier.Field{
Name: "Progress deadline",
Value: fmt.Sprintf("%vs", cd.GetProgressDeadlineSeconds()),
},
)
if cd.Spec.CanaryAnalysis.StepWeight > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: fmt.Sprintf("Weight step: %v max: %v",
cd.Spec.CanaryAnalysis.StepWeight,
cd.Spec.CanaryAnalysis.MaxWeight),
})
} else if len(cd.Spec.CanaryAnalysis.Match) > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: "A/B Testing",
})
} else if cd.Spec.CanaryAnalysis.Iterations > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: "Blue/Green",
})
}
}
err := c.notifier.Post(cd.Name, cd.Namespace, message, fields, warn)
if err != nil {
+43
View File
@@ -0,0 +1,43 @@
package notifier
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"time"
)
func postMessage(address string, payload interface{}) error {
data, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshalling notification payload failed %v", err)
}
b := bytes.NewBuffer(data)
req, err := http.NewRequest("POST", address, b)
if err != nil {
return err
}
req.Header.Set("Content-type", "application/json")
ctx, cancel := context.WithTimeout(req.Context(), 5*time.Second)
defer cancel()
res, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
return fmt.Errorf("sending notification failed %v", err)
}
defer res.Body.Close()
statusCode := res.StatusCode
if statusCode != 200 {
body, _ := ioutil.ReadAll(res.Body)
return fmt.Errorf("sending notification failed %v", string(body))
}
return nil
}
+26
View File
@@ -0,0 +1,26 @@
package notifier
type Factory struct {
URL string
Username string
Channel string
}
func NewFactory(URL string, username string, channel string) *Factory {
return &Factory{
URL: URL,
Channel: channel,
Username: username,
}
}
func (f Factory) Notifier(provider string) (Interface, error) {
switch {
case provider == "slack":
return NewSlack(f.URL, f.Username, f.Channel)
case provider == "msteams":
return NewMSTeams(f.URL)
}
return nil, nil
}
+10
View File
@@ -0,0 +1,10 @@
package notifier
type Interface interface {
Post(workload string, namespace string, message string, fields []Field, warn bool) error
}
type Field struct {
Name string
Value string
}
+9 -21
View File
@@ -1,12 +1,8 @@
package notifier
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
@@ -67,7 +63,7 @@ func NewSlack(hookURL string, username string, channel string) (*Slack, error) {
}
// Post Slack message
func (s *Slack) Post(workload string, namespace string, message string, fields []SlackField, warn bool) error {
func (s *Slack) Post(workload string, namespace string, message string, fields []Field, warn bool) error {
payload := SlackPayload{
Channel: s.Channel,
Username: s.Username,
@@ -78,32 +74,24 @@ func (s *Slack) Post(workload string, namespace string, message string, fields [
color = "danger"
}
sfields := make([]SlackField, len(fields))
for _, f := range fields {
sfields = append(sfields, SlackField{f.Name, f.Value, false})
}
a := SlackAttachment{
Color: color,
AuthorName: fmt.Sprintf("%s.%s", workload, namespace),
Text: message,
MrkdwnIn: []string{"text"},
Fields: fields,
Fields: sfields,
}
payload.Attachments = []SlackAttachment{a}
data, err := json.Marshal(payload)
err := postMessage(s.URL, payload)
if err != nil {
return fmt.Errorf("marshalling slack payload failed %v", err)
}
b := bytes.NewBuffer(data)
if res, err := http.Post(s.URL, "application/json", b); err != nil {
return fmt.Errorf("sending data to slack failed %v", err)
} else {
defer res.Body.Close()
statusCode := res.StatusCode
if statusCode != 200 {
body, _ := ioutil.ReadAll(res.Body)
return fmt.Errorf("sending data to slack failed %v", string(body))
}
return err
}
return nil
+77
View File
@@ -0,0 +1,77 @@
package notifier
import (
"fmt"
"net/url"
)
// MS Teams holds the incoming webhook URL
type MSTeams struct {
URL string
}
// MSTeamsPayload holds the message card data
type MSTeamsPayload struct {
Type string `json:"@type"`
Context string `json:"@context"`
ThemeColor string `json:"themeColor"`
Summary string `json:"summary"`
Sections []MSTeamsSection `json:"sections"`
}
// MSTeamsSection holds the canary analysis result
type MSTeamsSection struct {
ActivityTitle string `json:"activityTitle"`
ActivitySubtitle string `json:"activitySubtitle"`
Facts []MSTeamsField `json:"facts"`
}
type MSTeamsField struct {
Name string `json:"name"`
Value string `json:"value"`
}
// NewMSTeams validates the MS Teams URL and returns a MSTeams object
func NewMSTeams(hookURL string) (*MSTeams, error) {
_, err := url.ParseRequestURI(hookURL)
if err != nil {
return nil, fmt.Errorf("invalid MS Teams webhook URL %s", hookURL)
}
return &MSTeams{
URL: hookURL,
}, nil
}
// Post MS Teams message
func (s *MSTeams) Post(workload string, namespace string, message string, fields []Field, warn bool) error {
facts := make([]MSTeamsField, len(fields))
for _, f := range fields {
facts = append(facts, MSTeamsField{f.Name, f.Value})
}
payload := MSTeamsPayload{
Type: "MessageCard",
Context: "http://schema.org/extensions",
ThemeColor: "0076D7",
Summary: fmt.Sprintf("%s.%s", workload, namespace),
Sections: []MSTeamsSection{
{
ActivityTitle: message,
ActivitySubtitle: fmt.Sprintf("%s.%s", workload, namespace),
Facts: facts,
},
},
}
if warn {
payload.ThemeColor = "FF0000"
}
err := postMessage(s.URL, payload)
if err != nil {
return err
}
return nil
}