Merge pull request #429 from weaveworks/alerts

Implement canary alerts and alert providers

artifacts/flagger/account.yaml

@@ -44,6 +44,8 @@ rules:
       - canaries/status
       - metrictemplates
       - metrictemplates/status
+      - alertproviders
+      - alertproviders/status
     verbs: ["*"]
   - apiGroups:
       - networking.istio.io

artifacts/flagger/crd.yaml

@@ -591,3 +591,62 @@ spec:
             query:
               description: Query of this metric template
               type: string
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: alertproviders.flagger.app
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  group: flagger.app
+  version: v1beta1
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+  names:
+    plural: alertproviders
+    singular: alertprovider
+    kind: AlertProvider
+    categories:
+      - all
+  scope: Namespaced
+  subresources:
+    status: {}
+  additionalPrinterColumns:
+    - name: Type
+      type: string
+      JSONPath: .spec.type
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          oneOf:
+            - required:
+              - type
+              - address
+            - required:
+              - type
+              - secretRef
+          properties:
+            type:
+              description: Type of this provider
+              type: string
+              enum:
+                - slack
+                - msteams
+                - discord
+                - rocket
+            address:
+              description: Hook URL address of this provider
+              type: string
+            secretRef:
+              description: Kubernetes secret reference containing the provider address
+              type: object
+              required:
+                - name
+              properties:
+                name:
+                  description: Name of the Kubernetes secret
+                  type: string

charts/flagger/templates/crd.yaml

@@ -592,4 +592,63 @@ spec:
             query:
               description: Query of this metric template
               type: string
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: alertproviders.flagger.app
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  group: flagger.app
+  version: v1beta1
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+  names:
+    plural: alertproviders
+    singular: alertprovider
+    kind: AlertProvider
+    categories:
+      - all
+  scope: Namespaced
+  subresources:
+    status: {}
+  additionalPrinterColumns:
+    - name: Type
+      type: string
+      JSONPath: .spec.type
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          oneOf:
+            - required:
+              - type
+              - address
+            - required:
+              - type
+              - secretRef
+          properties:
+            type:
+              description: Type of this provider
+              type: string
+              enum:
+                - slack
+                - msteams
+                - discord
+                - rocket
+            address:
+              description: Hook URL address of this provider
+              type: string
+            secretRef:
+              description: Kubernetes secret reference containing the provider address
+              type: object
+              required:
+                - name
+              properties:
+                name:
+                  description: Name of the Kubernetes secret
+                  type: string
 {{- end }}

charts/flagger/templates/rbac.yaml

@@ -40,6 +40,8 @@ rules:
       - canaries/status
       - metrictemplates
       - metrictemplates/status
+      - alertproviders
+      - alertproviders/status
     verbs: ["*"]
   - apiGroups:
       - networking.istio.io

cmd/flagger/main.go

@@ -129,6 +129,7 @@ func main() {
 
 	verifyCRDs(flaggerClient, logger)
 	verifyKubernetesVersion(kubeClient, logger)
+	infos := startInformers(flaggerClient, logger, stopCh)
 
 	labels := strings.Split(selectorLabels, ",")
 	if len(labels) < 1 {
@@ -157,23 +158,6 @@ func main() {
 	// start HTTP server
 	go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
 
-	// start informers
-	flaggerInformerFactory := informers.NewSharedInformerFactoryWithOptions(flaggerClient, time.Second*30, informers.WithNamespace(namespace))
-
-	logger.Info("Waiting for canary informer cache to sync")
-	canaryInformer := flaggerInformerFactory.Flagger().V1beta1().Canaries()
-	go canaryInformer.Informer().Run(stopCh)
-	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, canaryInformer.Informer().HasSynced); !ok {
-		logger.Fatalf("failed to wait for cache to sync")
-	}
-
-	logger.Info("Waiting for metric template informer cache to sync")
-	metricInformer := flaggerInformerFactory.Flagger().V1beta1().MetricTemplates()
-	go metricInformer.Informer().Run(stopCh)
-	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, metricInformer.Informer().HasSynced); !ok {
-		logger.Fatalf("failed to wait for cache to sync")
-	}
-
 	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient)
 
 	var configTracker canary.Tracker
@@ -193,7 +177,7 @@ func main() {
 		kubeClient,
 		meshClient,
 		flaggerClient,
-		canaryInformer,
+		infos,
 		controlLoopInterval,
 		logger,
 		notifierClient,
@@ -237,6 +221,37 @@ func main() {
 	}
 }
 
+func startInformers(flaggerClient clientset.Interface, logger *zap.SugaredLogger, stopCh <-chan struct{}) controller.Informers {
+	flaggerInformerFactory := informers.NewSharedInformerFactoryWithOptions(flaggerClient, time.Second*30, informers.WithNamespace(namespace))
+
+	logger.Info("Waiting for canary informer cache to sync")
+	canaryInformer := flaggerInformerFactory.Flagger().V1beta1().Canaries()
+	go canaryInformer.Informer().Run(stopCh)
+	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, canaryInformer.Informer().HasSynced); !ok {
+		logger.Fatalf("failed to wait for cache to sync")
+	}
+
+	logger.Info("Waiting for metric template informer cache to sync")
+	metricInformer := flaggerInformerFactory.Flagger().V1beta1().MetricTemplates()
+	go metricInformer.Informer().Run(stopCh)
+	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, metricInformer.Informer().HasSynced); !ok {
+		logger.Fatalf("failed to wait for cache to sync")
+	}
+
+	logger.Info("Waiting for alert provider informer cache to sync")
+	alertInformer := flaggerInformerFactory.Flagger().V1beta1().AlertProviders()
+	go alertInformer.Informer().Run(stopCh)
+	if ok := cache.WaitForNamedCacheSync("flagger", stopCh, alertInformer.Informer().HasSynced); !ok {
+		logger.Fatalf("failed to wait for cache to sync")
+	}
+
+	return controller.Informers{
+		CanaryInformer: canaryInformer,
+		MetricInformer: metricInformer,
+		AlertInformer:  alertInformer,
+	}
+}
+
 func startLeaderElection(ctx context.Context, run func(), ns string, kubeClient kubernetes.Interface, logger *zap.SugaredLogger) {
 	configMapName := "flagger-leader-election"
 	id, err := os.Hostname()
@@ -322,6 +337,11 @@ func verifyCRDs(flaggerClient clientset.Interface, logger *zap.SugaredLogger) {
 	if err != nil {
 		logger.Fatalf("MetricTemplate CRD is not registered %v", err)
 	}
+
+	_, err = flaggerClient.FlaggerV1beta1().AlertProviders(namespace).List(metav1.ListOptions{Limit: 1})
+	if err != nil {
+		logger.Fatalf("AlertProvider CRD is not registered %v", err)
+	}
 }
 
 func verifyKubernetesVersion(kubeClient kubernetes.Interface, logger *zap.SugaredLogger) {

kustomize/base/flagger/crd.yaml

@@ -591,3 +591,62 @@ spec:
             query:
               description: Query of this metric template
               type: string
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: alertproviders.flagger.app
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  group: flagger.app
+  version: v1beta1
+  versions:
+    - name: v1beta1
+      served: true
+      storage: true
+  names:
+    plural: alertproviders
+    singular: alertprovider
+    kind: AlertProvider
+    categories:
+      - all
+  scope: Namespaced
+  subresources:
+    status: {}
+  additionalPrinterColumns:
+    - name: Type
+      type: string
+      JSONPath: .spec.type
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          oneOf:
+            - required:
+              - type
+              - address
+            - required:
+              - type
+              - secretRef
+          properties:
+            type:
+              description: Type of this provider
+              type: string
+              enum:
+                - slack
+                - msteams
+                - discord
+                - rocket
+            address:
+              description: Hook URL address of this provider
+              type: string
+            secretRef:
+              description: Kubernetes secret reference containing the provider address
+              type: object
+              required:
+                - name
+              properties:
+                name:
+                  description: Name of the Kubernetes secret
+                  type: string

kustomize/base/flagger/rbac.yaml

@@ -34,6 +34,8 @@ rules:
       - canaries/status
       - metrictemplates
       - metrictemplates/status
+      - alertproviders
+      - alertproviders/status
     verbs: ["*"]
   - apiGroups:
       - networking.istio.io

pkg/apis/flagger/v1beta1/alert.go

@@ -0,0 +1,95 @@
+/*
+Copyright The Flagger Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	AlertProviderKind = "AlertProvider"
+)
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// AlertProvider is the configuration of alerting for a specific provider
+type AlertProvider struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   AlertProviderSpec   `json:"spec"`
+	Status AlertProviderStatus `json:"status"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// AlertProviderList is a list of alert provider resources
+type AlertProviderList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []AlertProvider `json:"items"`
+}
+
+// AlertProviderSpec is the specification of the desired behavior of the AlertProvider
+type AlertProviderSpec struct {
+	// Type of provider
+	Type string `json:"type"`
+
+	// Alert channel for this provider
+	// +optional
+	Channel string `json:"channel,omitempty"`
+
+	// Bot username for this provider
+	// +optional
+	Username string `json:"username,omitempty"`
+
+	// HTTP(S) webhook address of this provider
+	// +optional
+	Address string `json:"address,omitempty"`
+
+	// Secret reference containing the provider webhook URL
+	// +optional
+	SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
+}
+
+type AlertProviderStatus struct {
+	// Conditions of this status
+	Conditions []AlertProviderCondition `json:"conditions,omitempty"`
+}
+
+type AlertProviderCondition struct {
+	// Type of this condition
+	Type string `json:"type"`
+
+	// Status of this condition
+	Status corev1.ConditionStatus `json:"status"`
+
+	// LastUpdateTime of this condition
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+
+	// LastTransitionTime of this condition
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+
+	// Reason for the current status of this condition
+	Reason string `json:"reason,omitempty"`
+
+	// Message associated with this condition
+	Message string `json:"message,omitempty"`
+}

pkg/apis/flagger/v1beta1/canary.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
-	hpav1 "k8s.io/api/autoscaling/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
@@ -36,7 +35,9 @@ const (
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// Canary is a specification for a Canary resource
+// Canary is the configuration for a canary release,
+// which automatically manages the bootstrap, analysis, traffic shifting,
+// promotion or rollback of an app revision
 type Canary struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -45,42 +46,6 @@ type Canary struct {
 	Status CanaryStatus `json:"status"`
 }
 
-// CanarySpec is the spec for a Canary resource
-type CanarySpec struct {
-	// if specified overwrites the -mesh-provider flag for this particular canary
-	// +optional
-	Provider string `json:"provider,omitempty"`
-
-	// if specified overwrites the -metrics-server flag for this particular canary
-	// +optional
-	MetricsServer string `json:"metricsServer,omitempty"`
-
-	// reference to target resource
-	TargetRef hpav1.CrossVersionObjectReference `json:"targetRef"`
-
-	// reference to autoscaling resource
-	// +optional
-	AutoscalerRef *hpav1.CrossVersionObjectReference `json:"autoscalerRef,omitempty"`
-
-	// reference to NGINX ingress resource
-	// +optional
-	IngressRef *hpav1.CrossVersionObjectReference `json:"ingressRef,omitempty"`
-
-	// virtual service spec
-	Service CanaryService `json:"service"`
-
-	// metrics, thresholds and webhooks spec
-	CanaryAnalysis CanaryAnalysis `json:"canaryAnalysis"`
-
-	// the maximum time in seconds for a canary deployment to make progress
-	// before it is considered to be failed. Defaults to ten minutes.
-	ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty"`
-
-	// promote the canary without analysing it
-	// +optional
-	SkipAnalysis bool `json:"skipAnalysis,omitempty"`
-}
-
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // CanaryList is a list of Canary resources
@@ -91,65 +56,207 @@ type CanaryList struct {
 	Items []Canary `json:"items"`
 }
 
-// CanaryService is used to create ClusterIP services
-// and service mesh or ingress routing objects
+// CanarySpec is the specification of the desired behavior of the Canary
+type CanarySpec struct {
+	// Provider overwrites the -mesh-provider flag for this particular canary
+	// +optional
+	Provider string `json:"provider,omitempty"`
+
+	// MetricsServer overwrites the -metrics-server flag for this particular canary
+	// +optional
+	MetricsServer string `json:"metricsServer,omitempty"`
+
+	// TargetRef references a target resource
+	TargetRef CrossNamespaceObjectReference `json:"targetRef"`
+
+	// AutoscalerRef references an autoscaling resource
+	// +optional
+	AutoscalerRef *CrossNamespaceObjectReference `json:"autoscalerRef,omitempty"`
+
+	// Reference to NGINX ingress resource
+	// +optional
+	IngressRef *CrossNamespaceObjectReference `json:"ingressRef,omitempty"`
+
+	// Service defines how ClusterIP services, service mesh or ingress routing objects are generated
+	Service CanaryService `json:"service"`
+
+	// CanaryAnalysis defines how the analysis should be performed
+	CanaryAnalysis CanaryAnalysis `json:"canaryAnalysis"`
+
+	// ProgressDeadlineSeconds represents the maximum time in seconds for a
+	// canary deployment to make progress before it is considered to be failed
+	// +optional
+	ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty"`
+
+	// SkipAnalysis promotes the canary without analysing it
+	// +optional
+	SkipAnalysis bool `json:"skipAnalysis,omitempty"`
+}
+
+// CanaryService defines how ClusterIP services, service mesh or ingress routing objects are generated
 type CanaryService struct {
-	Name          string             `json:"name,omitempty"`
-	Port          int32              `json:"port"`
-	PortName      string             `json:"portName,omitempty"`
-	TargetPort    intstr.IntOrString `json:"targetPort,omitempty"`
-	PortDiscovery bool               `json:"portDiscovery"`
-	Timeout       string             `json:"timeout,omitempty"`
-	// Istio
-	Gateways      []string                         `json:"gateways,omitempty"`
-	Hosts         []string                         `json:"hosts,omitempty"`
-	TrafficPolicy *istiov1alpha3.TrafficPolicy     `json:"trafficPolicy,omitempty"`
-	Match         []istiov1alpha3.HTTPMatchRequest `json:"match,omitempty"`
-	Rewrite       *istiov1alpha3.HTTPRewrite       `json:"rewrite,omitempty"`
-	Retries       *istiov1alpha3.HTTPRetry         `json:"retries,omitempty"`
-	Headers       *istiov1alpha3.Headers           `json:"headers,omitempty"`
-	CorsPolicy    *istiov1alpha3.CorsPolicy        `json:"corsPolicy,omitempty"`
-	// App Mesh
-	MeshName string   `json:"meshName,omitempty"`
+	// Name of the Kubernetes service generated by Flagger
+	// Defaults to CanarySpec.TargetRef.Name
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// Port of the generated Kubernetes service
+	Port int32 `json:"port"`
+
+	// Port name of the generated Kubernetes service
+	// Defaults to http
+	// +optional
+	PortName string `json:"portName,omitempty"`
+
+	// Target port number or name of the generated Kubernetes service
+	// Defaults to CanaryService.Port
+	// +optional
+	TargetPort intstr.IntOrString `json:"targetPort,omitempty"`
+
+	// PortDiscovery adds all container ports to the generated Kubernetes service
+	PortDiscovery bool `json:"portDiscovery"`
+
+	// Timeout of the HTTP or gRPC request
+	// +optional
+	Timeout string `json:"timeout,omitempty"`
+
+	// Gateways attached to the generated Istio virtual service
+	// Defaults to the internal mesh gateway
+	// +optional
+	Gateways []string `json:"gateways,omitempty"`
+
+	// Hosts attached to the generated Istio virtual service
+	// Defaults to the service name
+	// +optional
+	Hosts []string `json:"hosts,omitempty"`
+
+	// TrafficPolicy attached to the generated Istio destination rules
+	// +optional
+	TrafficPolicy *istiov1alpha3.TrafficPolicy `json:"trafficPolicy,omitempty"`
+
+	// URI match conditions for the generated service
+	// +optional
+	Match []istiov1alpha3.HTTPMatchRequest `json:"match,omitempty"`
+
+	// Rewrite HTTP URIs for the generated service
+	// +optional
+	Rewrite *istiov1alpha3.HTTPRewrite `json:"rewrite,omitempty"`
+
+	// Retries policy for the generated virtual service
+	// +optional
+	Retries *istiov1alpha3.HTTPRetry `json:"retries,omitempty"`
+
+	// Headers operations for the generated Istio virtual service
+	// +optional
+	Headers *istiov1alpha3.Headers `json:"headers,omitempty"`
+
+	// Cross-Origin Resource Sharing policy for the generated Istio virtual service
+	// +optional
+	CorsPolicy *istiov1alpha3.CorsPolicy `json:"corsPolicy,omitempty"`
+
+	// Mesh name of the generated App Mesh virtual nodes and virtual service
+	// +optional
+	MeshName string `json:"meshName,omitempty"`
+
+	// Backends of the generated App Mesh virtual nodes
+	// +optional
 	Backends []string `json:"backends,omitempty"`
 }
 
 // CanaryAnalysis is used to describe how the analysis should be done
 type CanaryAnalysis struct {
-	Interval   string                           `json:"interval"`
-	Threshold  int                              `json:"threshold"`
-	MaxWeight  int                              `json:"maxWeight"`
-	Mirror     bool                             `json:"mirror,omitempty"`
-	StepWeight int                              `json:"stepWeight"`
-	Metrics    []CanaryMetric                   `json:"metrics,omitempty"`
-	Webhooks   []CanaryWebhook                  `json:"webhooks,omitempty"`
-	Match      []istiov1alpha3.HTTPMatchRequest `json:"match,omitempty"`
-	Iterations int                              `json:"iterations,omitempty"`
+	// Schedule interval for this canary analysis
+	Interval string `json:"interval"`
+
+	// Number of checks to run for A/B Testing and Blue/Green
+	// +optional
+	Iterations int `json:"iterations,omitempty"`
+
+	//Enable traffic mirroring for Blue/Green
+	// +optional
+	Mirror bool `json:"mirror,omitempty"`
+
+	// Max traffic percentage routed to canary
+	// +optional
+	MaxWeight int `json:"maxWeight,omitempty"`
+
+	// Incremental traffic percentage step
+	// +optional
+	StepWeight int `json:"stepWeight,omitempty"`
+
+	// Max number of failed checks before the canary is terminated
+	Threshold int `json:"threshold"`
+
+	// Alert list for this canary analysis
+	Alerts []CanaryAlert `json:"alerts,omitempty"`
+
+	// Metric check list for this canary analysis
+	// +optional
+	Metrics []CanaryMetric `json:"metrics,omitempty"`
+
+	// Webhook list for this canary  analysis
+	// +optional
+	Webhooks []CanaryWebhook `json:"webhooks,omitempty"`
+
+	// A/B testing HTTP header match conditions
+	// +optional
+	Match []istiov1alpha3.HTTPMatchRequest `json:"match,omitempty"`
 }
 
 // CanaryMetric holds the reference to metrics used for canary analysis
 type CanaryMetric struct {
-	Name      string  `json:"name"`
-	Interval  string  `json:"interval,omitempty"`
+	// Name of the metric
+	Name string `json:"name"`
+
+	// Interval represents the windows size
+	Interval string `json:"interval,omitempty"`
+
+	// Max value accepted for this metric
 	Threshold float64 `json:"threshold"`
+
+	// Range value accepted for this metric
 	// +optional
 	ThresholdRange *CanaryThresholdRange `json:"thresholdRange,omitempty"`
+
+	// Prometheus query for this metric (deprecated in favor of TemplateRef)
 	// +optional
 	Query string `json:"query,omitempty"`
+
+	// TemplateRef references a metric template object
 	// +optional
-	TemplateRef *MetricTemplateRef `json:"templateRef,omitempty"`
+	TemplateRef *CrossNamespaceObjectReference `json:"templateRef,omitempty"`
 }
 
 // CanaryThresholdRange defines the range used for metrics validation
 type CanaryThresholdRange struct {
+	// Minimum value
+	// +optional
 	Min *float64 `json:"min,omitempty"`
+
+	// Maximum value
+	// +optional
 	Max *float64 `json:"max,omitempty"`
 }
 
-type MetricTemplateRef struct {
+// AlertSeverity defines alert filtering based on severity levels
+type AlertSeverity string
+
+const (
+	SeverityInfo  AlertSeverity = "info"
+	SeverityWarn  AlertSeverity = "warn"
+	SeverityError AlertSeverity = "error"
+)
+
+// CanaryAlert defines an alert for this canary
+type CanaryAlert struct {
+	// Name of the alert
 	Name string `json:"name"`
-	// +optional
-	Namespace string `json:"namespace,omitempty"`
+
+	// Severity level: info, warn, error (default info)
+	Severity AlertSeverity `json:"severity,omitempty"`
+
+	// Alert provider reference
+	ProviderRef CrossNamespaceObjectReference `json:"providerRef"`
 }
 
 // HookType can be pre, post or during rollout
@@ -168,26 +275,61 @@ const (
 	ConfirmPromotionHook HookType = "confirm-promotion"
 	// EventHook dispatches Flagger events to the specified endpoint
 	EventHook HookType = "event"
-	// RollbackHook rollback canary anaylysis if webhook returns HTTP 200
+	// RollbackHook rollback canary analysis if webhook returns HTTP 200
 	RollbackHook HookType = "rollback"
 )
 
 // CanaryWebhook holds the reference to external checks used for canary analysis
 type CanaryWebhook struct {
-	Type    HookType `json:"type"`
-	Name    string   `json:"name"`
-	URL     string   `json:"url"`
-	Timeout string   `json:"timeout"`
+	// Type of this webhook
+	Type HookType `json:"type"`
+
+	// Name of this webhook
+	Name string `json:"name"`
+
+	// URL address of this webhook
+	URL string `json:"url"`
+
+	// Request timeout for this webhook
+	Timeout string `json:"timeout"`
+
+	// Metadata (key-value pairs) for this webhook
 	// +optional
 	Metadata *map[string]string `json:"metadata,omitempty"`
 }
 
 // CanaryWebhookPayload holds the deployment info and metadata sent to webhooks
 type CanaryWebhookPayload struct {
-	Name      string            `json:"name"`
-	Namespace string            `json:"namespace"`
-	Phase     CanaryPhase       `json:"phase"`
-	Metadata  map[string]string `json:"metadata,omitempty"`
+	// Name of the canary
+	Name string `json:"name"`
+
+	// Namespace of the canary
+	Namespace string `json:"namespace"`
+
+	// Phase of the canary analysis
+	Phase CanaryPhase `json:"phase"`
+
+	// Metadata (key-value pairs) for this webhook
+	Metadata map[string]string `json:"metadata,omitempty"`
+}
+
+// CrossNamespaceObjectReference contains enough information to let you locate the
+// typed referenced object at cluster level
+type CrossNamespaceObjectReference struct {
+	// API version of the referent
+	// +optional
+	APIVersion string `json:"apiVersion,omitempty"`
+
+	// Kind of the referent
+	// +optional
+	Kind string `json:"kind,omitempty"`
+
+	// Name of the referent
+	Name string `json:"name"`
+
+	// Namespace of the referent
+	// +optional
+	Namespace string `json:"namespace,omitempty"`
 }
 
 // GetServiceNames returns the apex, primary and canary Kubernetes service names

pkg/apis/flagger/v1beta1/register.go

@@ -49,6 +49,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&CanaryList{},
 		&MetricTemplate{},
 		&MetricTemplateList{},
+		&AlertProvider{},
+		&AlertProviderList{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go

@@ -22,11 +22,133 @@ package v1beta1
 
 import (
 	v1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
-	v1 "k8s.io/api/autoscaling/v1"
-	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AlertProvider) DeepCopyInto(out *AlertProvider) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlertProvider.
+func (in *AlertProvider) DeepCopy() *AlertProvider {
+	if in == nil {
+		return nil
+	}
+	out := new(AlertProvider)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *AlertProvider) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AlertProviderCondition) DeepCopyInto(out *AlertProviderCondition) {
+	*out = *in
+	in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime)
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlertProviderCondition.
+func (in *AlertProviderCondition) DeepCopy() *AlertProviderCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(AlertProviderCondition)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AlertProviderList) DeepCopyInto(out *AlertProviderList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]AlertProvider, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlertProviderList.
+func (in *AlertProviderList) DeepCopy() *AlertProviderList {
+	if in == nil {
+		return nil
+	}
+	out := new(AlertProviderList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *AlertProviderList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AlertProviderSpec) DeepCopyInto(out *AlertProviderSpec) {
+	*out = *in
+	if in.SecretRef != nil {
+		in, out := &in.SecretRef, &out.SecretRef
+		*out = new(v1.LocalObjectReference)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlertProviderSpec.
+func (in *AlertProviderSpec) DeepCopy() *AlertProviderSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(AlertProviderSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AlertProviderStatus) DeepCopyInto(out *AlertProviderStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]AlertProviderCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlertProviderStatus.
+func (in *AlertProviderStatus) DeepCopy() *AlertProviderStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(AlertProviderStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Canary) DeepCopyInto(out *Canary) {
 	*out = *in
@@ -55,9 +177,31 @@ func (in *Canary) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CanaryAlert) DeepCopyInto(out *CanaryAlert) {
+	*out = *in
+	out.ProviderRef = in.ProviderRef
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryAlert.
+func (in *CanaryAlert) DeepCopy() *CanaryAlert {
+	if in == nil {
+		return nil
+	}
+	out := new(CanaryAlert)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CanaryAnalysis) DeepCopyInto(out *CanaryAnalysis) {
 	*out = *in
+	if in.Alerts != nil {
+		in, out := &in.Alerts, &out.Alerts
+		*out = make([]CanaryAlert, len(*in))
+		copy(*out, *in)
+	}
 	if in.Metrics != nil {
 		in, out := &in.Metrics, &out.Metrics
 		*out = make([]CanaryMetric, len(*in))
@@ -153,7 +297,7 @@ func (in *CanaryMetric) DeepCopyInto(out *CanaryMetric) {
 	}
 	if in.TemplateRef != nil {
 		in, out := &in.TemplateRef, &out.TemplateRef
-		*out = new(MetricTemplateRef)
+		*out = new(CrossNamespaceObjectReference)
 		**out = **in
 	}
 	return
@@ -239,12 +383,12 @@ func (in *CanarySpec) DeepCopyInto(out *CanarySpec) {
 	out.TargetRef = in.TargetRef
 	if in.AutoscalerRef != nil {
 		in, out := &in.AutoscalerRef, &out.AutoscalerRef
-		*out = new(v1.CrossVersionObjectReference)
+		*out = new(CrossNamespaceObjectReference)
 		**out = **in
 	}
 	if in.IngressRef != nil {
 		in, out := &in.IngressRef, &out.IngressRef
-		*out = new(v1.CrossVersionObjectReference)
+		*out = new(CrossNamespaceObjectReference)
 		**out = **in
 	}
 	in.Service.DeepCopyInto(&out.Service)
@@ -378,6 +522,22 @@ func (in *CanaryWebhookPayload) DeepCopy() *CanaryWebhookPayload {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CrossNamespaceObjectReference) DeepCopyInto(out *CrossNamespaceObjectReference) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CrossNamespaceObjectReference.
+func (in *CrossNamespaceObjectReference) DeepCopy() *CrossNamespaceObjectReference {
+	if in == nil {
+		return nil
+	}
+	out := new(CrossNamespaceObjectReference)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MetricTemplate) DeepCopyInto(out *MetricTemplate) {
 	*out = *in
@@ -478,7 +638,7 @@ func (in *MetricTemplateProvider) DeepCopyInto(out *MetricTemplateProvider) {
 	*out = *in
 	if in.SecretRef != nil {
 		in, out := &in.SecretRef, &out.SecretRef
-		*out = new(corev1.LocalObjectReference)
+		*out = new(v1.LocalObjectReference)
 		**out = **in
 	}
 	return
@@ -494,22 +654,6 @@ func (in *MetricTemplateProvider) DeepCopy() *MetricTemplateProvider {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *MetricTemplateRef) DeepCopyInto(out *MetricTemplateRef) {
-	*out = *in
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricTemplateRef.
-func (in *MetricTemplateRef) DeepCopy() *MetricTemplateRef {
-	if in == nil {
-		return nil
-	}
-	out := new(MetricTemplateRef)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *MetricTemplateSpec) DeepCopyInto(out *MetricTemplateSpec) {
 	*out = *in

pkg/canary/deployment_controller_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestCanaryDeployer_Sync(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -101,7 +101,7 @@ func TestCanaryDeployer_Sync(t *testing.T) {
 }
 
 func TestCanaryDeployer_IsNewSpec(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -124,7 +124,7 @@ func TestCanaryDeployer_IsNewSpec(t *testing.T) {
 }
 
 func TestCanaryDeployer_Promote(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -190,7 +190,7 @@ func TestCanaryDeployer_Promote(t *testing.T) {
 }
 
 func TestCanaryDeployer_IsReady(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Error("Expected primary readiness check to fail")
@@ -208,7 +208,7 @@ func TestCanaryDeployer_IsReady(t *testing.T) {
 }
 
 func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -230,7 +230,7 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
 }
 
 func TestCanaryDeployer_SetState(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -252,7 +252,7 @@ func TestCanaryDeployer_SetState(t *testing.T) {
 }
 
 func TestCanaryDeployer_SyncStatus(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -291,7 +291,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) {
 }
 
 func TestCanaryDeployer_Scale(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	err := mocks.deployer.Initialize(mocks.canary, true)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -310,7 +310,7 @@ func TestCanaryDeployer_Scale(t *testing.T) {
 }
 
 func TestCanaryDeployer_NoConfigTracking(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := newFixture()
 	mocks.deployer.configTracker = &NopTracker{}
 
 	err := mocks.deployer.Initialize(mocks.canary, true)

pkg/canary/fixture.go

@@ -4,7 +4,6 @@ import (
 	"github.com/weaveworks/flagger/pkg/logger"
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
-	hpav1 "k8s.io/api/autoscaling/v1"
 	hpav2 "k8s.io/api/autoscaling/v2beta1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -16,7 +15,7 @@ import (
 	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
 )
 
-type Mocks struct {
+type fixture struct {
 	canary        *flaggerv1.Canary
 	kubeClient    kubernetes.Interface
 	flaggerClient clientset.Interface
@@ -24,7 +23,7 @@ type Mocks struct {
 	logger        *zap.SugaredLogger
 }
 
-func SetupMocks() Mocks {
+func newFixture() fixture {
 	// init canary
 	canary := newTestCanary()
 	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
@@ -55,7 +54,7 @@ func SetupMocks() Mocks {
 		},
 	}
 
-	return Mocks{
+	return fixture{
 		canary:        canary,
 		deployer:      deployer,
 		logger:        logger,
@@ -181,12 +180,12 @@ func newTestCanary() *flaggerv1.Canary {
 			Name:      "podinfo",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 			},
-			AutoscalerRef: &hpav1.CrossVersionObjectReference{
+			AutoscalerRef: &flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "autoscaling/v2beta1",
 				Kind:       "HorizontalPodAutoscaler",
@@ -196,18 +195,6 @@ func newTestCanary() *flaggerv1.Canary {
 				Threshold:  10,
 				StepWeight: 10,
 				MaxWeight:  50,
-				Metrics: []flaggerv1.CanaryMetric{
-					{
-						Name:      "istio_requests_total",
-						Threshold: 99,
-						Interval:  "1m",
-					},
-					{
-						Name:      "istio_request_duration_seconds_bucket",
-						Threshold: 500,
-						Interval:  "1m",
-					},
-				},
 			},
 		},
 	}

pkg/client/clientset/versioned/typed/flagger/v1beta1/alertprovider.go

@@ -0,0 +1,191 @@
+/*
+Copyright The Flagger Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	"time"
+
+	v1beta1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	scheme "github.com/weaveworks/flagger/pkg/client/clientset/versioned/scheme"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	rest "k8s.io/client-go/rest"
+)
+
+// AlertProvidersGetter has a method to return a AlertProviderInterface.
+// A group's client should implement this interface.
+type AlertProvidersGetter interface {
+	AlertProviders(namespace string) AlertProviderInterface
+}
+
+// AlertProviderInterface has methods to work with AlertProvider resources.
+type AlertProviderInterface interface {
+	Create(*v1beta1.AlertProvider) (*v1beta1.AlertProvider, error)
+	Update(*v1beta1.AlertProvider) (*v1beta1.AlertProvider, error)
+	UpdateStatus(*v1beta1.AlertProvider) (*v1beta1.AlertProvider, error)
+	Delete(name string, options *v1.DeleteOptions) error
+	DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
+	Get(name string, options v1.GetOptions) (*v1beta1.AlertProvider, error)
+	List(opts v1.ListOptions) (*v1beta1.AlertProviderList, error)
+	Watch(opts v1.ListOptions) (watch.Interface, error)
+	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.AlertProvider, err error)
+	AlertProviderExpansion
+}
+
+// alertProviders implements AlertProviderInterface
+type alertProviders struct {
+	client rest.Interface
+	ns     string
+}
+
+// newAlertProviders returns a AlertProviders
+func newAlertProviders(c *FlaggerV1beta1Client, namespace string) *alertProviders {
+	return &alertProviders{
+		client: c.RESTClient(),
+		ns:     namespace,
+	}
+}
+
+// Get takes name of the alertProvider, and returns the corresponding alertProvider object, and an error if there is any.
+func (c *alertProviders) Get(name string, options v1.GetOptions) (result *v1beta1.AlertProvider, err error) {
+	result = &v1beta1.AlertProvider{}
+	err = c.client.Get().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		Name(name).
+		VersionedParams(&options, scheme.ParameterCodec).
+		Do().
+		Into(result)
+	return
+}
+
+// List takes label and field selectors, and returns the list of AlertProviders that match those selectors.
+func (c *alertProviders) List(opts v1.ListOptions) (result *v1beta1.AlertProviderList, err error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	result = &v1beta1.AlertProviderList{}
+	err = c.client.Get().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Do().
+		Into(result)
+	return
+}
+
+// Watch returns a watch.Interface that watches the requested alertProviders.
+func (c *alertProviders) Watch(opts v1.ListOptions) (watch.Interface, error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	opts.Watch = true
+	return c.client.Get().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Watch()
+}
+
+// Create takes the representation of a alertProvider and creates it.  Returns the server's representation of the alertProvider, and an error, if there is any.
+func (c *alertProviders) Create(alertProvider *v1beta1.AlertProvider) (result *v1beta1.AlertProvider, err error) {
+	result = &v1beta1.AlertProvider{}
+	err = c.client.Post().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		Body(alertProvider).
+		Do().
+		Into(result)
+	return
+}
+
+// Update takes the representation of a alertProvider and updates it. Returns the server's representation of the alertProvider, and an error, if there is any.
+func (c *alertProviders) Update(alertProvider *v1beta1.AlertProvider) (result *v1beta1.AlertProvider, err error) {
+	result = &v1beta1.AlertProvider{}
+	err = c.client.Put().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		Name(alertProvider.Name).
+		Body(alertProvider).
+		Do().
+		Into(result)
+	return
+}
+
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+
+func (c *alertProviders) UpdateStatus(alertProvider *v1beta1.AlertProvider) (result *v1beta1.AlertProvider, err error) {
+	result = &v1beta1.AlertProvider{}
+	err = c.client.Put().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		Name(alertProvider.Name).
+		SubResource("status").
+		Body(alertProvider).
+		Do().
+		Into(result)
+	return
+}
+
+// Delete takes name of the alertProvider and deletes it. Returns an error if one occurs.
+func (c *alertProviders) Delete(name string, options *v1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		Name(name).
+		Body(options).
+		Do().
+		Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *alertProviders) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+	var timeout time.Duration
+	if listOptions.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second
+	}
+	return c.client.Delete().
+		Namespace(c.ns).
+		Resource("alertproviders").
+		VersionedParams(&listOptions, scheme.ParameterCodec).
+		Timeout(timeout).
+		Body(options).
+		Do().
+		Error()
+}
+
+// Patch applies the patch and returns the patched alertProvider.
+func (c *alertProviders) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.AlertProvider, err error) {
+	result = &v1beta1.AlertProvider{}
+	err = c.client.Patch(pt).
+		Namespace(c.ns).
+		Resource("alertproviders").
+		SubResource(subresources...).
+		Name(name).
+		Body(data).
+		Do().
+		Into(result)
+	return
+}

pkg/client/clientset/versioned/typed/flagger/v1beta1/fake/fake_alertprovider.go

@@ -0,0 +1,140 @@
+/*
+Copyright The Flagger Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	v1beta1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+)
+
+// FakeAlertProviders implements AlertProviderInterface
+type FakeAlertProviders struct {
+	Fake *FakeFlaggerV1beta1
+	ns   string
+}
+
+var alertprovidersResource = schema.GroupVersionResource{Group: "flagger.app", Version: "v1beta1", Resource: "alertproviders"}
+
+var alertprovidersKind = schema.GroupVersionKind{Group: "flagger.app", Version: "v1beta1", Kind: "AlertProvider"}
+
+// Get takes name of the alertProvider, and returns the corresponding alertProvider object, and an error if there is any.
+func (c *FakeAlertProviders) Get(name string, options v1.GetOptions) (result *v1beta1.AlertProvider, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewGetAction(alertprovidersResource, c.ns, name), &v1beta1.AlertProvider{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.AlertProvider), err
+}
+
+// List takes label and field selectors, and returns the list of AlertProviders that match those selectors.
+func (c *FakeAlertProviders) List(opts v1.ListOptions) (result *v1beta1.AlertProviderList, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewListAction(alertprovidersResource, alertprovidersKind, c.ns, opts), &v1beta1.AlertProviderList{})
+
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &v1beta1.AlertProviderList{ListMeta: obj.(*v1beta1.AlertProviderList).ListMeta}
+	for _, item := range obj.(*v1beta1.AlertProviderList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested alertProviders.
+func (c *FakeAlertProviders) Watch(opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewWatchAction(alertprovidersResource, c.ns, opts))
+
+}
+
+// Create takes the representation of a alertProvider and creates it.  Returns the server's representation of the alertProvider, and an error, if there is any.
+func (c *FakeAlertProviders) Create(alertProvider *v1beta1.AlertProvider) (result *v1beta1.AlertProvider, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewCreateAction(alertprovidersResource, c.ns, alertProvider), &v1beta1.AlertProvider{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.AlertProvider), err
+}
+
+// Update takes the representation of a alertProvider and updates it. Returns the server's representation of the alertProvider, and an error, if there is any.
+func (c *FakeAlertProviders) Update(alertProvider *v1beta1.AlertProvider) (result *v1beta1.AlertProvider, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewUpdateAction(alertprovidersResource, c.ns, alertProvider), &v1beta1.AlertProvider{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.AlertProvider), err
+}
+
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+func (c *FakeAlertProviders) UpdateStatus(alertProvider *v1beta1.AlertProvider) (*v1beta1.AlertProvider, error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewUpdateSubresourceAction(alertprovidersResource, "status", c.ns, alertProvider), &v1beta1.AlertProvider{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.AlertProvider), err
+}
+
+// Delete takes name of the alertProvider and deletes it. Returns an error if one occurs.
+func (c *FakeAlertProviders) Delete(name string, options *v1.DeleteOptions) error {
+	_, err := c.Fake.
+		Invokes(testing.NewDeleteAction(alertprovidersResource, c.ns, name), &v1beta1.AlertProvider{})
+
+	return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeAlertProviders) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+	action := testing.NewDeleteCollectionAction(alertprovidersResource, c.ns, listOptions)
+
+	_, err := c.Fake.Invokes(action, &v1beta1.AlertProviderList{})
+	return err
+}
+
+// Patch applies the patch and returns the patched alertProvider.
+func (c *FakeAlertProviders) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1beta1.AlertProvider, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewPatchSubresourceAction(alertprovidersResource, c.ns, name, pt, data, subresources...), &v1beta1.AlertProvider{})
+
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1beta1.AlertProvider), err
+}

pkg/client/clientset/versioned/typed/flagger/v1beta1/fake/fake_flagger_client.go

@@ -28,6 +28,10 @@ type FakeFlaggerV1beta1 struct {
 	*testing.Fake
 }
 
+func (c *FakeFlaggerV1beta1) AlertProviders(namespace string) v1beta1.AlertProviderInterface {
+	return &FakeAlertProviders{c, namespace}
+}
+
 func (c *FakeFlaggerV1beta1) Canaries(namespace string) v1beta1.CanaryInterface {
 	return &FakeCanaries{c, namespace}
 }

pkg/client/clientset/versioned/typed/flagger/v1beta1/flagger_client.go

@@ -26,6 +26,7 @@ import (
 
 type FlaggerV1beta1Interface interface {
 	RESTClient() rest.Interface
+	AlertProvidersGetter
 	CanariesGetter
 	MetricTemplatesGetter
 }
@@ -35,6 +36,10 @@ type FlaggerV1beta1Client struct {
 	restClient rest.Interface
 }
 
+func (c *FlaggerV1beta1Client) AlertProviders(namespace string) AlertProviderInterface {
+	return newAlertProviders(c, namespace)
+}
+
 func (c *FlaggerV1beta1Client) Canaries(namespace string) CanaryInterface {
 	return newCanaries(c, namespace)
 }

pkg/client/clientset/versioned/typed/flagger/v1beta1/generated_expansion.go

@@ -18,6 +18,8 @@ limitations under the License.
 
 package v1beta1
 
+type AlertProviderExpansion interface{}
+
 type CanaryExpansion interface{}
 
 type MetricTemplateExpansion interface{}

pkg/client/informers/externalversions/flagger/v1beta1/alertprovider.go

@@ -0,0 +1,89 @@
+/*
+Copyright The Flagger Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	time "time"
+
+	flaggerv1beta1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	versioned "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
+	internalinterfaces "github.com/weaveworks/flagger/pkg/client/informers/externalversions/internalinterfaces"
+	v1beta1 "github.com/weaveworks/flagger/pkg/client/listers/flagger/v1beta1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+)
+
+// AlertProviderInformer provides access to a shared informer and lister for
+// AlertProviders.
+type AlertProviderInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1beta1.AlertProviderLister
+}
+
+type alertProviderInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+	namespace        string
+}
+
+// NewAlertProviderInformer constructs a new informer for AlertProvider type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewAlertProviderInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredAlertProviderInformer(client, namespace, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredAlertProviderInformer constructs a new informer for AlertProvider type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredAlertProviderInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.FlaggerV1beta1().AlertProviders(namespace).List(options)
+			},
+			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.FlaggerV1beta1().AlertProviders(namespace).Watch(options)
+			},
+		},
+		&flaggerv1beta1.AlertProvider{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *alertProviderInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredAlertProviderInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *alertProviderInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&flaggerv1beta1.AlertProvider{}, f.defaultInformer)
+}
+
+func (f *alertProviderInformer) Lister() v1beta1.AlertProviderLister {
+	return v1beta1.NewAlertProviderLister(f.Informer().GetIndexer())
+}

pkg/client/informers/externalversions/flagger/v1beta1/interface.go

@@ -24,6 +24,8 @@ import (
 
 // Interface provides access to all the informers in this group version.
 type Interface interface {
+	// AlertProviders returns a AlertProviderInformer.
+	AlertProviders() AlertProviderInformer
 	// Canaries returns a CanaryInformer.
 	Canaries() CanaryInformer
 	// MetricTemplates returns a MetricTemplateInformer.
@@ -41,6 +43,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList
 	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
 }
 
+// AlertProviders returns a AlertProviderInformer.
+func (v *version) AlertProviders() AlertProviderInformer {
+	return &alertProviderInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
+}
+
 // Canaries returns a CanaryInformer.
 func (v *version) Canaries() CanaryInformer {
 	return &canaryInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}

pkg/client/informers/externalversions/generic.go

@@ -66,6 +66,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Appmesh().V1beta1().VirtualServices().Informer()}, nil
 
 		// Group=flagger.app, Version=v1beta1
+	case flaggerv1beta1.SchemeGroupVersion.WithResource("alertproviders"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Flagger().V1beta1().AlertProviders().Informer()}, nil
 	case flaggerv1beta1.SchemeGroupVersion.WithResource("canaries"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Flagger().V1beta1().Canaries().Informer()}, nil
 	case flaggerv1beta1.SchemeGroupVersion.WithResource("metrictemplates"):

pkg/client/listers/flagger/v1beta1/alertprovider.go

@@ -0,0 +1,94 @@
+/*
+Copyright The Flagger Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	v1beta1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+)
+
+// AlertProviderLister helps list AlertProviders.
+type AlertProviderLister interface {
+	// List lists all AlertProviders in the indexer.
+	List(selector labels.Selector) (ret []*v1beta1.AlertProvider, err error)
+	// AlertProviders returns an object that can list and get AlertProviders.
+	AlertProviders(namespace string) AlertProviderNamespaceLister
+	AlertProviderListerExpansion
+}
+
+// alertProviderLister implements the AlertProviderLister interface.
+type alertProviderLister struct {
+	indexer cache.Indexer
+}
+
+// NewAlertProviderLister returns a new AlertProviderLister.
+func NewAlertProviderLister(indexer cache.Indexer) AlertProviderLister {
+	return &alertProviderLister{indexer: indexer}
+}
+
+// List lists all AlertProviders in the indexer.
+func (s *alertProviderLister) List(selector labels.Selector) (ret []*v1beta1.AlertProvider, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1beta1.AlertProvider))
+	})
+	return ret, err
+}
+
+// AlertProviders returns an object that can list and get AlertProviders.
+func (s *alertProviderLister) AlertProviders(namespace string) AlertProviderNamespaceLister {
+	return alertProviderNamespaceLister{indexer: s.indexer, namespace: namespace}
+}
+
+// AlertProviderNamespaceLister helps list and get AlertProviders.
+type AlertProviderNamespaceLister interface {
+	// List lists all AlertProviders in the indexer for a given namespace.
+	List(selector labels.Selector) (ret []*v1beta1.AlertProvider, err error)
+	// Get retrieves the AlertProvider from the indexer for a given namespace and name.
+	Get(name string) (*v1beta1.AlertProvider, error)
+	AlertProviderNamespaceListerExpansion
+}
+
+// alertProviderNamespaceLister implements the AlertProviderNamespaceLister
+// interface.
+type alertProviderNamespaceLister struct {
+	indexer   cache.Indexer
+	namespace string
+}
+
+// List lists all AlertProviders in the indexer for a given namespace.
+func (s alertProviderNamespaceLister) List(selector labels.Selector) (ret []*v1beta1.AlertProvider, err error) {
+	err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1beta1.AlertProvider))
+	})
+	return ret, err
+}
+
+// Get retrieves the AlertProvider from the indexer for a given namespace and name.
+func (s alertProviderNamespaceLister) Get(name string) (*v1beta1.AlertProvider, error) {
+	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(v1beta1.Resource("alertprovider"), name)
+	}
+	return obj.(*v1beta1.AlertProvider), nil
+}

pkg/client/listers/flagger/v1beta1/expansion_generated.go

@@ -18,6 +18,14 @@ limitations under the License.
 
 package v1beta1
 
+// AlertProviderListerExpansion allows custom methods to be added to
+// AlertProviderLister.
+type AlertProviderListerExpansion interface{}
+
+// AlertProviderNamespaceListerExpansion allows custom methods to be added to
+// AlertProviderNamespaceLister.
+type AlertProviderNamespaceListerExpansion interface{}
+
 // CanaryListerExpansion allows custom methods to be added to
 // CanaryLister.
 type CanaryListerExpansion interface{}

pkg/controller/controller.go

@@ -24,7 +24,6 @@ import (
 	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	flaggerscheme "github.com/weaveworks/flagger/pkg/client/clientset/versioned/scheme"
 	flaggerinformers "github.com/weaveworks/flagger/pkg/client/informers/externalversions/flagger/v1beta1"
-	flaggerlisters "github.com/weaveworks/flagger/pkg/client/listers/flagger/v1beta1"
 	"github.com/weaveworks/flagger/pkg/metrics"
 	"github.com/weaveworks/flagger/pkg/metrics/observers"
 	"github.com/weaveworks/flagger/pkg/notifier"
@@ -35,31 +34,37 @@ const controllerAgentName = "flagger"
 
 // Controller is managing the canary objects and schedules canary deployments
 type Controller struct {
-	kubeClient      kubernetes.Interface
-	istioClient     clientset.Interface
-	flaggerClient   clientset.Interface
-	flaggerLister   flaggerlisters.CanaryLister
-	flaggerSynced   cache.InformerSynced
-	flaggerWindow   time.Duration
-	workqueue       workqueue.RateLimitingInterface
-	eventRecorder   record.EventRecorder
-	logger          *zap.SugaredLogger
-	canaries        *sync.Map
-	jobs            map[string]CanaryJob
-	recorder        metrics.Recorder
-	notifier        notifier.Interface
-	canaryFactory   *canary.Factory
-	routerFactory   *router.Factory
-	observerFactory *observers.Factory
-	meshProvider    string
-	eventWebhook    string
+	kubeClient       kubernetes.Interface
+	istioClient      clientset.Interface
+	flaggerClient    clientset.Interface
+	flaggerInformers Informers
+	flaggerSynced    cache.InformerSynced
+	flaggerWindow    time.Duration
+	workqueue        workqueue.RateLimitingInterface
+	eventRecorder    record.EventRecorder
+	logger           *zap.SugaredLogger
+	canaries         *sync.Map
+	jobs             map[string]CanaryJob
+	recorder         metrics.Recorder
+	notifier         notifier.Interface
+	canaryFactory    *canary.Factory
+	routerFactory    *router.Factory
+	observerFactory  *observers.Factory
+	meshProvider     string
+	eventWebhook     string
+}
+
+type Informers struct {
+	CanaryInformer flaggerinformers.CanaryInformer
+	MetricInformer flaggerinformers.MetricTemplateInformer
+	AlertInformer  flaggerinformers.AlertProviderInformer
 }
 
 func NewController(
 	kubeClient kubernetes.Interface,
 	istioClient clientset.Interface,
 	flaggerClient clientset.Interface,
-	flaggerInformer flaggerinformers.CanaryInformer,
+	flaggerInformers Informers,
 	flaggerWindow time.Duration,
 	logger *zap.SugaredLogger,
 	notifier notifier.Interface,
@@ -83,27 +88,27 @@ func NewController(
 	recorder.SetInfo(version, meshProvider)
 
 	ctrl := &Controller{
-		kubeClient:      kubeClient,
-		istioClient:     istioClient,
-		flaggerClient:   flaggerClient,
-		flaggerLister:   flaggerInformer.Lister(),
-		flaggerSynced:   flaggerInformer.Informer().HasSynced,
-		workqueue:       workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
-		eventRecorder:   eventRecorder,
-		logger:          logger,
-		canaries:        new(sync.Map),
-		jobs:            map[string]CanaryJob{},
-		flaggerWindow:   flaggerWindow,
-		observerFactory: observerFactory,
-		recorder:        recorder,
-		notifier:        notifier,
-		canaryFactory:   canaryFactory,
-		routerFactory:   routerFactory,
-		meshProvider:    meshProvider,
-		eventWebhook:    eventWebhook,
+		kubeClient:       kubeClient,
+		istioClient:      istioClient,
+		flaggerClient:    flaggerClient,
+		flaggerInformers: flaggerInformers,
+		flaggerSynced:    flaggerInformers.CanaryInformer.Informer().HasSynced,
+		workqueue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
+		eventRecorder:    eventRecorder,
+		logger:           logger,
+		canaries:         new(sync.Map),
+		jobs:             map[string]CanaryJob{},
+		flaggerWindow:    flaggerWindow,
+		observerFactory:  observerFactory,
+		recorder:         recorder,
+		notifier:         notifier,
+		canaryFactory:    canaryFactory,
+		routerFactory:    routerFactory,
+		meshProvider:     meshProvider,
+		eventWebhook:     eventWebhook,
 	}
 
-	flaggerInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+	flaggerInformers.CanaryInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: ctrl.enqueue,
 		UpdateFunc: func(old, new interface{}) {
 			oldCanary, ok := checkCustomResourceType(old, logger)
@@ -209,7 +214,7 @@ func (c *Controller) syncHandler(key string) error {
 		utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key))
 		return nil
 	}
-	cd, err := c.flaggerLister.Canaries(namespace).Get(name)
+	cd, err := c.flaggerInformers.CanaryInformer.Lister().Canaries(namespace).Get(name)
 	if errors.IsNotFound(err) {
 		utilruntime.HandleError(fmt.Errorf("%s in work queue no longer exists", key))
 		return nil
@@ -256,94 +261,6 @@ func checkCustomResourceType(obj interface{}, logger *zap.SugaredLogger) (flagge
 	return *roll, true
 }
 
-func (c *Controller) sendEventToWebhook(r *flaggerv1.Canary, eventtype, template string, args []interface{}) {
-	webhookOverride := false
-	if len(r.Spec.CanaryAnalysis.Webhooks) > 0 {
-		for _, canaryWebhook := range r.Spec.CanaryAnalysis.Webhooks {
-			if canaryWebhook.Type == flaggerv1.EventHook {
-				webhookOverride = true
-				err := CallEventWebhook(r, canaryWebhook.URL, fmt.Sprintf(template, args...), eventtype)
-				if err != nil {
-					c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf("error sending event to webhook: %s", err)
-				}
-			}
-		}
-	}
-
-	if c.eventWebhook != "" && !webhookOverride {
-		err := CallEventWebhook(r, c.eventWebhook, fmt.Sprintf(template, args...), eventtype)
-		if err != nil {
-			c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf("error sending event to webhook: %s", err)
-		}
-	}
-}
-
-func (c *Controller) recordEventInfof(r *flaggerv1.Canary, template string, args ...interface{}) {
-	c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...)
-	c.eventRecorder.Event(r, corev1.EventTypeNormal, "Synced", fmt.Sprintf(template, args...))
-	c.sendEventToWebhook(r, corev1.EventTypeNormal, template, args)
-}
-
-func (c *Controller) recordEventErrorf(r *flaggerv1.Canary, template string, args ...interface{}) {
-	c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf(template, args...)
-	c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...))
-	c.sendEventToWebhook(r, corev1.EventTypeWarning, template, args)
-}
-
-func (c *Controller) recordEventWarningf(r *flaggerv1.Canary, template string, args ...interface{}) {
-	c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...)
-	c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...))
-	c.sendEventToWebhook(r, corev1.EventTypeWarning, template, args)
-}
-
-func (c *Controller) sendNotification(cd *flaggerv1.Canary, message string, metadata bool, warn bool) {
-	if c.notifier == nil {
-		return
-	}
-
-	var fields []notifier.Field
-
-	if metadata {
-		fields = append(fields,
-			notifier.Field{
-				Name:  "Target",
-				Value: fmt.Sprintf("%s/%s.%s", cd.Spec.TargetRef.Kind, cd.Spec.TargetRef.Name, cd.Namespace),
-			},
-			notifier.Field{
-				Name:  "Failed checks threshold",
-				Value: fmt.Sprintf("%v", cd.Spec.CanaryAnalysis.Threshold),
-			},
-			notifier.Field{
-				Name:  "Progress deadline",
-				Value: fmt.Sprintf("%vs", cd.GetProgressDeadlineSeconds()),
-			},
-		)
-
-		if cd.Spec.CanaryAnalysis.StepWeight > 0 {
-			fields = append(fields, notifier.Field{
-				Name: "Traffic routing",
-				Value: fmt.Sprintf("Weight step: %v max: %v",
-					cd.Spec.CanaryAnalysis.StepWeight,
-					cd.Spec.CanaryAnalysis.MaxWeight),
-			})
-		} else if len(cd.Spec.CanaryAnalysis.Match) > 0 {
-			fields = append(fields, notifier.Field{
-				Name:  "Traffic routing",
-				Value: "A/B Testing",
-			})
-		} else if cd.Spec.CanaryAnalysis.Iterations > 0 {
-			fields = append(fields, notifier.Field{
-				Name:  "Traffic routing",
-				Value: "Blue/Green",
-			})
-		}
-	}
-	err := c.notifier.Post(cd.Name, cd.Namespace, message, fields, warn)
-	if err != nil {
-		c.logger.Error(err)
-	}
-}
-
 func int32p(i int32) *int32 {
 	return &i
 }

pkg/controller/events.go

@@ -0,0 +1,191 @@
+package controller
+
+import (
+	"fmt"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	corev1 "k8s.io/api/core/v1"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	"github.com/weaveworks/flagger/pkg/notifier"
+)
+
+func (c *Controller) recordEventInfof(r *flaggerv1.Canary, template string, args ...interface{}) {
+	c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...)
+	c.eventRecorder.Event(r, corev1.EventTypeNormal, "Synced", fmt.Sprintf(template, args...))
+	c.sendEventToWebhook(r, corev1.EventTypeNormal, template, args)
+}
+
+func (c *Controller) recordEventErrorf(r *flaggerv1.Canary, template string, args ...interface{}) {
+	c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf(template, args...)
+	c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...))
+	c.sendEventToWebhook(r, corev1.EventTypeWarning, template, args)
+}
+
+func (c *Controller) recordEventWarningf(r *flaggerv1.Canary, template string, args ...interface{}) {
+	c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Infof(template, args...)
+	c.eventRecorder.Event(r, corev1.EventTypeWarning, "Synced", fmt.Sprintf(template, args...))
+	c.sendEventToWebhook(r, corev1.EventTypeWarning, template, args)
+}
+
+func (c *Controller) sendEventToWebhook(r *flaggerv1.Canary, eventType, template string, args []interface{}) {
+	webhookOverride := false
+	if len(r.Spec.CanaryAnalysis.Webhooks) > 0 {
+		for _, canaryWebhook := range r.Spec.CanaryAnalysis.Webhooks {
+			if canaryWebhook.Type == flaggerv1.EventHook {
+				webhookOverride = true
+				err := CallEventWebhook(r, canaryWebhook.URL, fmt.Sprintf(template, args...), eventType)
+				if err != nil {
+					c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf("error sending event to webhook: %s", err)
+				}
+			}
+		}
+	}
+
+	if c.eventWebhook != "" && !webhookOverride {
+		err := CallEventWebhook(r, c.eventWebhook, fmt.Sprintf(template, args...), eventType)
+		if err != nil {
+			c.logger.With("canary", fmt.Sprintf("%s.%s", r.Name, r.Namespace)).Errorf("error sending event to webhook: %s", err)
+		}
+	}
+}
+
+func (c *Controller) alert(canary *flaggerv1.Canary, message string, metadata bool, severity flaggerv1.AlertSeverity) {
+	if c.notifier == nil && len(canary.Spec.CanaryAnalysis.Alerts) == 0 {
+		return
+	}
+
+	var fields []notifier.Field
+	if metadata {
+		fields = alertMetadata(canary)
+	}
+
+	// send alert with the global notifier
+	if len(canary.Spec.CanaryAnalysis.Alerts) == 0 {
+		err := c.notifier.Post(canary.Name, canary.Namespace, message, fields, string(severity))
+		if err != nil {
+			c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+				Errorf("alert can't be sent: %v", err)
+			return
+		}
+		return
+	}
+
+	// send canary alerts
+	for _, alert := range canary.Spec.CanaryAnalysis.Alerts {
+		// determine if alert should be sent based on severity level
+		shouldAlert := false
+		if alert.Severity == flaggerv1.SeverityInfo {
+			shouldAlert = true
+		} else {
+			if severity == alert.Severity {
+				shouldAlert = true
+			}
+			if severity == flaggerv1.SeverityWarn && alert.Severity == flaggerv1.SeverityError {
+				shouldAlert = true
+			}
+		}
+		if !shouldAlert {
+			continue
+		}
+
+		// determine alert provider namespace
+		providerNamespace := canary.GetNamespace()
+		if alert.ProviderRef.Namespace != "" {
+			providerNamespace = alert.ProviderRef.Namespace
+		}
+
+		// find alert provider
+		provider, err := c.flaggerInformers.AlertInformer.Lister().AlertProviders(providerNamespace).Get(alert.ProviderRef.Name)
+		if err != nil {
+			c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+				Errorf("alert provider %s.%s error: %v", alert.ProviderRef.Name, providerNamespace, err)
+			continue
+		}
+
+		// set hook URL address
+		url := provider.Spec.Address
+
+		// extract address from secret
+		if provider.Spec.SecretRef != nil {
+			secret, err := c.kubeClient.CoreV1().Secrets(providerNamespace).Get(provider.Spec.SecretRef.Name, metav1.GetOptions{})
+			if err != nil {
+				c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+					Errorf("alert provider %s.%s secretRef error: %v", alert.ProviderRef.Name, providerNamespace, err)
+				continue
+			}
+			if address, ok := secret.Data["address"]; ok {
+				url = string(address)
+			} else {
+				c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+					Errorf("alert provider %s.%s secret does not contain an address", alert.ProviderRef.Name, providerNamespace)
+				continue
+			}
+		}
+
+		// set defaults
+		username := "flagger"
+		if provider.Spec.Username != "" {
+			username = provider.Spec.Username
+		}
+		channel := "general"
+		if provider.Spec.Channel != "" {
+			channel = provider.Spec.Channel
+		}
+
+		// create notifier based on provider type
+		f := notifier.NewFactory(url, username, channel)
+		n, err := f.Notifier(provider.Spec.Type)
+		if err != nil {
+			c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+				Errorf("alert provider %s.%s error: %v", alert.ProviderRef.Name, providerNamespace, err)
+			continue
+		}
+
+		// send alert
+		err = n.Post(canary.Name, canary.Namespace, message, fields, string(severity))
+		if err != nil {
+			c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+				Errorf("alert provider $s.%s send error: %v", alert.ProviderRef.Name, providerNamespace, err)
+		}
+
+	}
+}
+
+func alertMetadata(canary *flaggerv1.Canary) []notifier.Field {
+	var fields []notifier.Field
+	fields = append(fields,
+		notifier.Field{
+			Name:  "Target",
+			Value: fmt.Sprintf("%s/%s.%s", canary.Spec.TargetRef.Kind, canary.Spec.TargetRef.Name, canary.Namespace),
+		},
+		notifier.Field{
+			Name:  "Failed checks threshold",
+			Value: fmt.Sprintf("%v", canary.Spec.CanaryAnalysis.Threshold),
+		},
+		notifier.Field{
+			Name:  "Progress deadline",
+			Value: fmt.Sprintf("%vs", canary.GetProgressDeadlineSeconds()),
+		},
+	)
+
+	if canary.Spec.CanaryAnalysis.StepWeight > 0 {
+		fields = append(fields, notifier.Field{
+			Name: "Traffic routing",
+			Value: fmt.Sprintf("Weight step: %v max: %v",
+				canary.Spec.CanaryAnalysis.StepWeight,
+				canary.Spec.CanaryAnalysis.MaxWeight),
+		})
+	} else if len(canary.Spec.CanaryAnalysis.Match) > 0 {
+		fields = append(fields, notifier.Field{
+			Name:  "Traffic routing",
+			Value: "A/B Testing",
+		})
+	} else if canary.Spec.CanaryAnalysis.Iterations > 0 {
+		fields = append(fields, notifier.Field{
+			Name:  "Traffic routing",
+			Value: "Blue/Green",
+		})
+	}
+	return fields
+}

pkg/controller/fixture.go

@@ -7,7 +7,6 @@ import (
 
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
-	hpav1 "k8s.io/api/autoscaling/v1"
 	hpav2 "k8s.io/api/autoscaling/v2beta1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -34,7 +33,7 @@ var (
 	noResyncPeriodFunc = func() time.Duration { return 0 }
 )
 
-type Mocks struct {
+type fixture struct {
 	canary        *flaggerv1.Canary
 	kubeClient    kubernetes.Interface
 	meshClient    clientset.Interface
@@ -45,30 +44,42 @@ type Mocks struct {
 	router        router.Interface
 }
 
-func SetupMocks(c *flaggerv1.Canary) Mocks {
+func newFixture(c *flaggerv1.Canary) fixture {
 	if c == nil {
 		c = newTestCanary()
 	}
-	flaggerClient := fakeFlagger.NewSimpleClientset(c, newTestMetricTemplate())
 
-	// init kube clientset and register mock objects
+	// init Flagger clientset and register objects
+	flaggerClient := fakeFlagger.NewSimpleClientset(
+		c,
+		newTestMetricTemplate(),
+		newTestAlertProvider(),
+	)
+
+	// init Kubernetes clientset and register objects
 	kubeClient := fake.NewSimpleClientset(
 		newTestDeployment(),
 		newTestService(),
 		newTestHPA(),
-		NewTestConfigMap(),
-		NewTestConfigMapEnv(),
-		NewTestConfigMapVol(),
-		NewTestSecret(),
-		NewTestSecretEnv(),
-		NewTestSecretVol(),
+		newTestConfigMap(),
+		newTestConfigMapEnv(),
+		newTestConfigMapVol(),
+		newTestSecret(),
+		newTestSecretEnv(),
+		newTestSecretVol(),
+		newTestAlertProviderSecret(),
 	)
 
 	logger, _ := logger.NewLogger("debug")
 
 	// init controller
-	flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc())
-	flaggerInformer := flaggerInformerFactory.Flagger().V1beta1().Canaries()
+	flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, 0)
+
+	fi := Informers{
+		CanaryInformer: flaggerInformerFactory.Flagger().V1beta1().Canaries(),
+		MetricInformer: flaggerInformerFactory.Flagger().V1beta1().MetricTemplates(),
+		AlertInformer:  flaggerInformerFactory.Flagger().V1beta1().AlertProviders(),
+	}
 
 	// init router
 	rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient)
@@ -85,26 +96,29 @@ func SetupMocks(c *flaggerv1.Canary) Mocks {
 	canaryFactory := canary.NewFactory(kubeClient, flaggerClient, configTracker, []string{"app", "name"}, logger)
 
 	ctrl := &Controller{
-		kubeClient:      kubeClient,
-		istioClient:     flaggerClient,
-		flaggerClient:   flaggerClient,
-		flaggerLister:   flaggerInformer.Lister(),
-		flaggerSynced:   flaggerInformer.Informer().HasSynced,
-		workqueue:       workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
-		eventRecorder:   &record.FakeRecorder{},
-		logger:          logger,
-		canaries:        new(sync.Map),
-		flaggerWindow:   time.Second,
-		canaryFactory:   canaryFactory,
-		observerFactory: observerFactory,
-		recorder:        metrics.NewRecorder(controllerAgentName, false),
-		routerFactory:   rf,
+		kubeClient:       kubeClient,
+		istioClient:      flaggerClient,
+		flaggerClient:    flaggerClient,
+		flaggerInformers: fi,
+		flaggerSynced:    fi.CanaryInformer.Informer().HasSynced,
+		workqueue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
+		eventRecorder:    &record.FakeRecorder{},
+		logger:           logger,
+		canaries:         new(sync.Map),
+		flaggerWindow:    time.Second,
+		canaryFactory:    canaryFactory,
+		observerFactory:  observerFactory,
+		recorder:         metrics.NewRecorder(controllerAgentName, false),
+		routerFactory:    rf,
 	}
 	ctrl.flaggerSynced = alwaysReady
+	ctrl.flaggerInformers.CanaryInformer.Informer().GetIndexer().Add(c)
+	ctrl.flaggerInformers.MetricInformer.Informer().GetIndexer().Add(newTestMetricTemplate())
+	ctrl.flaggerInformers.AlertInformer.Informer().GetIndexer().Add(newTestAlertProvider())
 
 	meshRouter := rf.MeshRouter("istio")
 
-	return Mocks{
+	return fixture{
 		canary:        c,
 		deployer:      canaryFactory.Controller("Deployment"),
 		logger:        logger,
@@ -116,7 +130,7 @@ func SetupMocks(c *flaggerv1.Canary) Mocks {
 	}
 }
 
-func NewTestConfigMap() *corev1.ConfigMap {
+func newTestConfigMap() *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -129,7 +143,7 @@ func NewTestConfigMap() *corev1.ConfigMap {
 	}
 }
 
-func NewTestConfigMapV2() *corev1.ConfigMap {
+func newTestConfigMapV2() *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -143,7 +157,7 @@ func NewTestConfigMapV2() *corev1.ConfigMap {
 	}
 }
 
-func NewTestConfigMapEnv() *corev1.ConfigMap {
+func newTestConfigMapEnv() *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -156,7 +170,7 @@ func NewTestConfigMapEnv() *corev1.ConfigMap {
 	}
 }
 
-func NewTestConfigMapVol() *corev1.ConfigMap {
+func newTestConfigMapVol() *corev1.ConfigMap {
 	return &corev1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -169,7 +183,7 @@ func NewTestConfigMapVol() *corev1.ConfigMap {
 	}
 }
 
-func NewTestSecret() *corev1.Secret {
+func newTestSecret() *corev1.Secret {
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -185,7 +199,7 @@ func NewTestSecret() *corev1.Secret {
 	}
 }
 
-func NewTestSecretV2() *corev1.Secret {
+func newTestSecretV2() *corev1.Secret {
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -201,7 +215,7 @@ func NewTestSecretV2() *corev1.Secret {
 	}
 }
 
-func NewTestSecretEnv() *corev1.Secret {
+func newTestSecretEnv() *corev1.Secret {
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -215,7 +229,7 @@ func NewTestSecretEnv() *corev1.Secret {
 	}
 }
 
-func NewTestSecretVol() *corev1.Secret {
+func newTestSecretVol() *corev1.Secret {
 	return &corev1.Secret{
 		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
@@ -237,12 +251,12 @@ func newTestCanary() *flaggerv1.Canary {
 			Name:      "podinfo",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 			},
-			AutoscalerRef: &hpav1.CrossVersionObjectReference{
+			AutoscalerRef: &flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "autoscaling/v2beta1",
 				Kind:       "HorizontalPodAutoscaler",
@@ -273,7 +287,7 @@ func newTestCanary() *flaggerv1.Canary {
 							Max: toFloatPtr(100),
 						},
 						Interval: "1m",
-						TemplateRef: &flaggerv1.MetricTemplateRef{
+						TemplateRef: &flaggerv1.CrossNamespaceObjectReference{
 							Name:      "envoy",
 							Namespace: "default",
 						},
@@ -304,12 +318,12 @@ func newTestCanaryAB() *flaggerv1.Canary {
 			Name:      "podinfo",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 			},
-			AutoscalerRef: &hpav1.CrossVersionObjectReference{
+			AutoscalerRef: &flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "autoscaling/v2beta1",
 				Kind:       "HorizontalPodAutoscaler",
@@ -696,3 +710,34 @@ func newTestMetricTemplate() *flaggerv1.MetricTemplate {
 	}
 	return template
 }
+
+func newTestAlertProviderSecret() *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "alert-secret",
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"address": []byte("http://mock.slack"),
+		},
+	}
+}
+
+func newTestAlertProvider() *flaggerv1.AlertProvider {
+	return &flaggerv1.AlertProvider{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "slack",
+		},
+		Spec: flaggerv1.AlertProviderSpec{
+			Type:    "slack",
+			Address: "http://fake.slack",
+			SecretRef: &corev1.LocalObjectReference{
+				Name: "alert-secret",
+			},
+		},
+	}
+}

pkg/controller/scheduler.go

@@ -230,7 +230,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		cd.Status.Phase == flaggerv1.CanaryPhaseWaiting {
 		if ok := c.runRollbackHooks(cd, cd.Status.Phase); ok {
 			c.recordEventWarningf(cd, "Rolling back %s.%s manual webhook invoked", cd.Name, cd.Namespace)
-			c.sendNotification(cd, "Rolling back manual webhook invoked", false, true)
+			c.alert(cd, "Rolling back manual webhook invoked", false, flaggerv1.SeverityWarn)
 			c.rollback(cd, canaryController, meshRouter)
 			return
 		}
@@ -271,8 +271,8 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
 		c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded)
 		c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
-		c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
-			false, false)
+		c.alert(cd, "Canary analysis completed successfully, promotion finished.",
+			false, flaggerv1.SeverityInfo)
 		return
 	}
 
@@ -282,8 +282,8 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		if !retriable {
 			c.recordEventWarningf(cd, "Rolling back %s.%s progress deadline exceeded %v",
 				cd.Name, cd.Namespace, err)
-			c.sendNotification(cd, fmt.Sprintf("Progress deadline exceeded %v", err),
-				false, true)
+			c.alert(cd, fmt.Sprintf("Progress deadline exceeded %v", err),
+				false, flaggerv1.SeverityError)
 		}
 		c.rollback(cd, canaryController, meshRouter)
 		return
@@ -569,8 +569,8 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll
 	c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseSucceeded)
 	c.recordEventInfof(canary, "Promotion completed! Canary analysis was skipped for %s.%s",
 		canary.Spec.TargetRef.Name, canary.Namespace)
-	c.sendNotification(canary, "Canary analysis was skipped, promotion finished.",
-		false, false)
+	c.alert(canary, "Canary analysis was skipped, promotion finished.",
+		false, flaggerv1.SeverityInfo)
 
 	return true
 }
@@ -617,8 +617,8 @@ func (c *Controller) checkCanaryStatus(canary *flaggerv1.Canary, canaryControlle
 		}
 		c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseInitialized)
 		c.recordEventInfof(canary, "Initialization done! %s.%s", canary.Name, canary.Namespace)
-		c.sendNotification(canary, "New deployment detected, initialization completed.",
-			true, false)
+		c.alert(canary, "New deployment detected, initialization completed.",
+			true, flaggerv1.SeverityInfo)
 		return false
 	}
 
@@ -626,8 +626,8 @@ func (c *Controller) checkCanaryStatus(canary *flaggerv1.Canary, canaryControlle
 		canaryPhaseProgressing := canary.DeepCopy()
 		canaryPhaseProgressing.Status.Phase = flaggerv1.CanaryPhaseProgressing
 		c.recordEventInfof(canaryPhaseProgressing, "New revision detected! Scaling up %s.%s", canaryPhaseProgressing.Spec.TargetRef.Name, canaryPhaseProgressing.Namespace)
-		c.sendNotification(canaryPhaseProgressing, "New revision detected, starting canary analysis.",
-			true, false)
+		c.alert(canaryPhaseProgressing, "New revision detected, starting canary analysis.",
+			true, flaggerv1.SeverityInfo)
 
 		if err := canaryController.ScaleFromZero(canary); err != nil {
 			c.recordEventErrorf(canary, "%v", err)
@@ -666,7 +666,7 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary, canaryCont
 					}
 					c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s",
 						canary.Name, canary.Namespace, webhook.Name)
-					c.sendNotification(canary, "Canary is waiting for approval.", false, false)
+					c.alert(canary, "Canary is waiting for approval.", false, flaggerv1.SeverityWarn)
 				}
 				return false
 			} else {
@@ -691,7 +691,7 @@ func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool {
 			if err != nil {
 				c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for promotion approval %s",
 					canary.Name, canary.Namespace, webhook.Name)
-				c.sendNotification(canary, "Canary promotion is waiting for approval.", false, false)
+				c.alert(canary, "Canary promotion is waiting for approval.", false, flaggerv1.SeverityWarn)
 				return false
 			} else {
 				c.recordEventInfof(canary, "Confirm-promotion check %s passed", webhook.Name)
@@ -921,7 +921,7 @@ func (c *Controller) runMetricChecks(canary *flaggerv1.Canary) bool {
 				namespace = metric.TemplateRef.Namespace
 			}
 
-			template, err := c.flaggerClient.FlaggerV1beta1().MetricTemplates(namespace).Get(metric.TemplateRef.Name, metav1.GetOptions{})
+			template, err := c.flaggerInformers.MetricInformer.Lister().MetricTemplates(namespace).Get(metric.TemplateRef.Name)
 			if err != nil {
 				c.recordEventErrorf(canary, "Metric template %s.%s error: %v", metric.TemplateRef.Name, namespace, err)
 				return false
@@ -1010,8 +1010,8 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
 	if canary.Status.FailedChecks >= canary.Spec.CanaryAnalysis.Threshold {
 		c.recordEventWarningf(canary, "Rolling back %s.%s failed checks threshold reached %v",
 			canary.Name, canary.Namespace, canary.Status.FailedChecks)
-		c.sendNotification(canary, fmt.Sprintf("Failed checks threshold reached %v", canary.Status.FailedChecks),
-			false, true)
+		c.alert(canary, fmt.Sprintf("Failed checks threshold reached %v", canary.Status.FailedChecks),
+			false, flaggerv1.SeverityError)
 	}
 
 	// route all traffic back to primary

pkg/controller/scheduler_svc_test.go

@@ -3,14 +3,13 @@ package controller
 import (
 	"testing"
 
-	hpav1 "k8s.io/api/autoscaling/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
 )
 
 func TestScheduler_ServicePromotion(t *testing.T) {
-	mocks := SetupMocks(newTestServiceCanary())
+	mocks := newFixture(newTestServiceCanary())
 
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
@@ -135,7 +134,7 @@ func newTestServiceCanary() *flaggerv1.Canary {
 			Name:      "podinfo",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "core/v1",
 				Kind:       "Service",
@@ -149,13 +148,13 @@ func newTestServiceCanary() *flaggerv1.Canary {
 				MaxWeight:  50,
 				Metrics: []flaggerv1.CanaryMetric{
 					{
-						Name:      "istio_requests_total",
+						Name:      "request-success-rate",
 						Threshold: 99,
 						Interval:  "1m",
 					},
 					{
-						Name:      "istio_request_duration_seconds_bucket",
-						Threshold: 500,
+						Name:      "request-duration",
+						Threshold: 500000,
 						Interval:  "1m",
 					},
 				},

pkg/controller/scheduler_test.go

@@ -1,17 +1,22 @@
 package controller
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	"github.com/weaveworks/flagger/pkg/notifier"
 )
 
 func TestScheduler_Init(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	_, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
@@ -21,7 +26,7 @@ func TestScheduler_Init(t *testing.T) {
 }
 
 func TestScheduler_NewRevision(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// update
@@ -45,7 +50,7 @@ func TestScheduler_NewRevision(t *testing.T) {
 }
 
 func TestScheduler_Rollback(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -99,7 +104,7 @@ func TestScheduler_Rollback(t *testing.T) {
 }
 
 func TestScheduler_SkipAnalysis(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -140,7 +145,7 @@ func TestScheduler_SkipAnalysis(t *testing.T) {
 }
 
 func TestScheduler_NewRevisionReset(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -202,7 +207,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 }
 
 func TestScheduler_Promotion(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
@@ -227,13 +232,13 @@ func TestScheduler_Promotion(t *testing.T) {
 	// detect pod spec changes
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	config2 := NewTestConfigMapV2()
+	config2 := newTestConfigMapV2()
 	_, err = mocks.kubeClient.CoreV1().ConfigMaps("default").Update(config2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	secret2 := NewTestSecretV2()
+	secret2 := newTestSecretV2()
 	_, err = mocks.kubeClient.CoreV1().Secrets("default").Update(secret2)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -353,7 +358,7 @@ func TestScheduler_Promotion(t *testing.T) {
 }
 
 func TestScheduler_Mirroring(t *testing.T) {
-	mocks := SetupMocks(newTestCanaryMirror())
+	mocks := newFixture(newTestCanaryMirror())
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -411,7 +416,7 @@ func TestScheduler_Mirroring(t *testing.T) {
 }
 
 func TestScheduler_ABTesting(t *testing.T) {
-	mocks := SetupMocks(newTestCanaryAB())
+	mocks := newFixture(newTestCanaryAB())
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -499,7 +504,7 @@ func TestScheduler_ABTesting(t *testing.T) {
 }
 
 func TestScheduler_PortDiscovery(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 
 	// enable port discovery
 	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get("podinfo", metav1.GetOptions{})
@@ -543,7 +548,7 @@ func TestScheduler_PortDiscovery(t *testing.T) {
 }
 
 func TestScheduler_TargetPortNumber(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 
 	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
@@ -588,7 +593,7 @@ func TestScheduler_TargetPortNumber(t *testing.T) {
 }
 
 func TestScheduler_TargetPortName(t *testing.T) {
-	mocks := SetupMocks(nil)
+	mocks := newFixture(nil)
 
 	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
@@ -631,3 +636,53 @@ func TestScheduler_TargetPortName(t *testing.T) {
 
 	}
 }
+
+func TestScheduler_Alerts(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var payload = notifier.SlackPayload{}
+		err = json.Unmarshal(b, &payload)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if payload.Attachments[0].AuthorName != "podinfo.default" {
+			t.Fatal("wrong author name")
+		}
+	}))
+	defer ts.Close()
+
+	canary := newTestCanary()
+	canary.Spec.CanaryAnalysis.Alerts = []flaggerv1.CanaryAlert{
+		{
+			Name:     "slack-dev",
+			Severity: "info",
+			ProviderRef: flaggerv1.CrossNamespaceObjectReference{
+				Name:      "slack",
+				Namespace: "default",
+			},
+		},
+		{
+			Name:     "slack-prod",
+			Severity: "info",
+			ProviderRef: flaggerv1.CrossNamespaceObjectReference{
+				Name: "slack",
+			},
+		},
+	}
+	mocks := newFixture(canary)
+
+	secret := newTestAlertProviderSecret()
+	secret.Data = map[string][]byte{
+		"address": []byte(ts.URL),
+	}
+	_, err := mocks.kubeClient.CoreV1().Secrets("default").Update(secret)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// init canary and send alerts
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+}

pkg/notifier/discord.go

@@ -0,0 +1,81 @@
+package notifier
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"path"
+	"strings"
+)
+
+// Discord holds the hook URL
+type Discord struct {
+	URL      string
+	Username string
+	Channel  string
+}
+
+// NewDiscord validates the URL and returns a Discord object
+func NewDiscord(hookURL string, username string, channel string) (*Discord, error) {
+	webhook, err := url.ParseRequestURI(hookURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid Discord hook URL %s", hookURL)
+	}
+
+	// use Slack formatting
+	// https://birdie0.github.io/discord-webhooks-guide/other/slack_formatting.html
+	if !strings.HasSuffix(hookURL, "/slack") {
+		webhook.Path = path.Join(webhook.Path, "slack")
+		hookURL = webhook.String()
+	}
+
+	if username == "" {
+		return nil, errors.New("empty Discord username")
+	}
+
+	if channel == "" {
+		return nil, errors.New("empty Discord channel")
+	}
+
+	return &Discord{
+		Channel:  channel,
+		URL:      hookURL,
+		Username: username,
+	}, nil
+}
+
+// Post Discord message
+func (s *Discord) Post(workload string, namespace string, message string, fields []Field, severity string) error {
+	payload := SlackPayload{
+		Channel:   s.Channel,
+		Username:  s.Username,
+		IconEmoji: ":rocket:",
+	}
+
+	color := "good"
+	if severity == "error" {
+		color = "danger"
+	}
+
+	sfields := make([]SlackField, 0, len(fields))
+	for _, f := range fields {
+		sfields = append(sfields, SlackField{f.Name, f.Value, false})
+	}
+
+	a := SlackAttachment{
+		Color:      color,
+		AuthorName: fmt.Sprintf("%s.%s", workload, namespace),
+		Text:       message,
+		MrkdwnIn:   []string{"text"},
+		Fields:     sfields,
+	}
+
+	payload.Attachments = []SlackAttachment{a}
+
+	err := postMessage(s.URL, payload)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/notifier/discord_test.go

@@ -0,0 +1,49 @@
+package notifier
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestDiscord_Post(t *testing.T) {
+	fields := []Field{
+		{Name: "name1", Value: "value1"},
+		{Name: "name2", Value: "value2"},
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var payload = SlackPayload{}
+		err = json.Unmarshal(b, &payload)
+
+		if payload.Attachments[0].AuthorName != "podinfo.test" {
+			t.Fatal("wrong author name")
+		}
+
+		if len(payload.Attachments[0].Fields) != len(fields) {
+			t.Fatal("wrong facts")
+		}
+	}))
+	defer ts.Close()
+
+	discord, err := NewDiscord(ts.URL, "test", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !strings.HasSuffix(discord.URL, "/slack") {
+		t.Error("Invalid Discord URL, expected to have /slack prefix")
+	}
+
+	err = discord.Post("podinfo", "test", "test", fields, "warn")
+	if err != nil {
+		t.Fatal(err)
+	}
+}

pkg/notifier/factory.go

@@ -1,5 +1,7 @@
 package notifier
 
+import "fmt"
+
 type Factory struct {
 	URL      string
 	Username string
@@ -18,9 +20,13 @@ func (f Factory) Notifier(provider string) (Interface, error) {
 	switch {
 	case provider == "slack":
 		return NewSlack(f.URL, f.Username, f.Channel)
+	case provider == "discord":
+		return NewDiscord(f.URL, f.Username, f.Channel)
+	case provider == "rocket":
+		return NewRocket(f.URL, f.Username, f.Channel)
 	case provider == "msteams":
 		return NewMSTeams(f.URL)
 	}
 
-	return nil, nil
+	return nil, fmt.Errorf("provider %s not supported", provider)
 }

pkg/notifier/notifier.go

@@ -1,7 +1,7 @@
 package notifier
 
 type Interface interface {
-	Post(workload string, namespace string, message string, fields []Field, warn bool) error
+	Post(workload string, namespace string, message string, fields []Field, severity string) error
 }
 
 type Field struct {

pkg/notifier/rocket.go

@@ -0,0 +1,72 @@
+package notifier
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+)
+
+// Rocket holds the hook URL
+type Rocket struct {
+	URL      string
+	Username string
+	Channel  string
+}
+
+// NewRocket validates the Rocket URL and returns a Rocket object
+func NewRocket(hookURL string, username string, channel string) (*Rocket, error) {
+	_, err := url.ParseRequestURI(hookURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid Rocket hook URL %s", hookURL)
+	}
+
+	if username == "" {
+		return nil, errors.New("empty Rocket username")
+	}
+
+	if channel == "" {
+		return nil, errors.New("empty Rocket channel")
+	}
+
+	return &Rocket{
+		Channel:  channel,
+		URL:      hookURL,
+		Username: username,
+	}, nil
+}
+
+// Post Rocket message
+func (s *Rocket) Post(workload string, namespace string, message string, fields []Field, severity string) error {
+	payload := SlackPayload{
+		Channel:   s.Channel,
+		Username:  s.Username,
+		IconEmoji: ":rocket:",
+	}
+
+	color := "#0076D7"
+	if severity == "error" {
+		color = "#FF0000"
+	}
+
+	sfields := make([]SlackField, 0, len(fields))
+	for _, f := range fields {
+		sfields = append(sfields, SlackField{f.Name, f.Value, false})
+	}
+
+	a := SlackAttachment{
+		Color:      color,
+		AuthorName: fmt.Sprintf("%s.%s", workload, namespace),
+		Text:       message,
+		MrkdwnIn:   []string{"text"},
+		Fields:     sfields,
+	}
+
+	payload.Attachments = []SlackAttachment{a}
+
+	err := postMessage(s.URL, payload)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/notifier/rocket_test.go

@@ -0,0 +1,44 @@
+package notifier
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestSlack_Post(t *testing.T) {
+	fields := []Field{
+		{Name: "name1", Value: "value1"},
+		{Name: "name2", Value: "value2"},
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var payload = SlackPayload{}
+		err = json.Unmarshal(b, &payload)
+
+		if payload.Attachments[0].AuthorName != "podinfo.test" {
+			t.Fatal("wrong author name")
+		}
+
+		if len(payload.Attachments[0].Fields) != len(fields) {
+			t.Fatal("wrong facts")
+		}
+	}))
+	defer ts.Close()
+
+	slack, err := NewSlack(ts.URL, "test", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = slack.Post("podinfo", "test", "test", fields, "error")
+	if err != nil {
+		t.Fatal(err)
+	}
+}

pkg/notifier/slack.go

@@ -8,10 +8,9 @@ import (
 
 // Slack holds the hook URL
 type Slack struct {
-	URL       string
-	Username  string
-	Channel   string
-	IconEmoji string
+	URL      string
+	Username string
+	Channel  string
 }
 
 // SlackPayload holds the channel and attachments
@@ -55,22 +54,22 @@ func NewSlack(hookURL string, username string, channel string) (*Slack, error) {
 	}
 
 	return &Slack{
-		Channel:   channel,
-		URL:       hookURL,
-		Username:  username,
-		IconEmoji: ":rocket:",
+		Channel:  channel,
+		URL:      hookURL,
+		Username: username,
 	}, nil
 }
 
 // Post Slack message
-func (s *Slack) Post(workload string, namespace string, message string, fields []Field, warn bool) error {
+func (s *Slack) Post(workload string, namespace string, message string, fields []Field, severity string) error {
 	payload := SlackPayload{
-		Channel:  s.Channel,
-		Username: s.Username,
+		Channel:   s.Channel,
+		Username:  s.Username,
+		IconEmoji: ":rocket:",
 	}
 
 	color := "good"
-	if warn {
+	if severity == "error" {
 		color = "danger"
 	}
 

pkg/notifier/slack_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 )
 
-func TestSlack_Post(t *testing.T) {
+func TestRocket_Post(t *testing.T) {
 	fields := []Field{
 		{Name: "name1", Value: "value1"},
 		{Name: "name2", Value: "value2"},
@@ -27,17 +27,17 @@ func TestSlack_Post(t *testing.T) {
 		}
 
 		if len(payload.Attachments[0].Fields) != len(fields) {
-			t.Fatal("wrong facts")
+			t.Fatal("wrong field")
 		}
 	}))
 	defer ts.Close()
 
-	slack, err := NewSlack(ts.URL, "test", "test")
+	rocket, err := NewRocket(ts.URL, "test", "test")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	err = slack.Post("podinfo", "test", "test", fields, true)
+	err = rocket.Post("podinfo", "test", "test", fields, "error")
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/notifier/teams.go

@@ -44,7 +44,7 @@ func NewMSTeams(hookURL string) (*MSTeams, error) {
 }
 
 // Post MS Teams message
-func (s *MSTeams) Post(workload string, namespace string, message string, fields []Field, warn bool) error {
+func (s *MSTeams) Post(workload string, namespace string, message string, fields []Field, severity string) error {
 	facts := make([]MSTeamsField, 0, len(fields))
 	for _, f := range fields {
 		facts = append(facts, MSTeamsField{f.Name, f.Value})
@@ -64,7 +64,7 @@ func (s *MSTeams) Post(workload string, namespace string, message string, fields
 		},
 	}
 
-	if warn {
+	if severity == "error" {
 		payload.ThemeColor = "FF0000"
 	}
 

pkg/notifier/teams_test.go

@@ -37,7 +37,7 @@ func TestTeams_Post(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = teams.Post("podinfo", "test", "test", fields, true)
+	err = teams.Post("podinfo", "test", "test", fields, "info")
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/router/appmesh_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestAppmeshRouter_Reconcile(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &AppMeshRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -151,7 +151,7 @@ func TestAppmeshRouter_Reconcile(t *testing.T) {
 }
 
 func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &AppMeshRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -188,7 +188,7 @@ func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
 }
 
 func TestAppmeshRouter_ABTest(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &AppMeshRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -230,7 +230,7 @@ func TestAppmeshRouter_ABTest(t *testing.T) {
 }
 
 func TestAppmeshRouter_Gateway(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &AppMeshRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,

pkg/router/contour_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestContourRouter_Reconcile(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &ContourRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -116,7 +116,7 @@ func TestContourRouter_Reconcile(t *testing.T) {
 }
 
 func TestContourRouter_Routes(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &ContourRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,

pkg/router/gloo_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestGlooRouter_Sync(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &GlooRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -43,7 +43,7 @@ func TestGlooRouter_Sync(t *testing.T) {
 }
 
 func TestGlooRouter_SetRoutes(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &GlooRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -100,7 +100,7 @@ func TestGlooRouter_SetRoutes(t *testing.T) {
 }
 
 func TestGlooRouter_GetRoutes(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &GlooRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,

pkg/router/ingress_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestIngressRouter_Reconcile(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IngressRouter{
 		logger:            mocks.logger,
 		kubeClient:        mocks.kubeClient,
@@ -44,7 +44,7 @@ func TestIngressRouter_Reconcile(t *testing.T) {
 }
 
 func TestIngressRouter_GetSetRoutes(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IngressRouter{
 		logger:            mocks.logger,
 		kubeClient:        mocks.kubeClient,

pkg/router/istio_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestIstioRouter_Sync(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IstioRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -108,7 +108,7 @@ func TestIstioRouter_Sync(t *testing.T) {
 }
 
 func TestIstioRouter_SetRoutes(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IstioRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -213,7 +213,7 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
 }
 
 func TestIstioRouter_GetRoutes(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IstioRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -309,7 +309,7 @@ func TestIstioRouter_GetRoutes(t *testing.T) {
 }
 
 func TestIstioRouter_HTTPRequestHeaders(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IstioRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -338,7 +338,7 @@ func TestIstioRouter_HTTPRequestHeaders(t *testing.T) {
 }
 
 func TestIstioRouter_CORS(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IstioRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,
@@ -371,7 +371,7 @@ func TestIstioRouter_CORS(t *testing.T) {
 }
 
 func TestIstioRouter_ABTest(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &IstioRouter{
 		logger:        mocks.logger,
 		flaggerClient: mocks.flaggerClient,

pkg/router/kubernetes_deployment_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestServiceRouter_Create(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &KubernetesDeploymentRouter{
 		kubeClient:    mocks.kubeClient,
 		flaggerClient: mocks.flaggerClient,
@@ -52,7 +52,7 @@ func TestServiceRouter_Create(t *testing.T) {
 }
 
 func TestServiceRouter_Update(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &KubernetesDeploymentRouter{
 		kubeClient:    mocks.kubeClient,
 		flaggerClient: mocks.flaggerClient,
@@ -103,7 +103,7 @@ func TestServiceRouter_Update(t *testing.T) {
 }
 
 func TestServiceRouter_Undo(t *testing.T) {
-	mocks := setupfakeClients()
+	mocks := newFixture()
 	router := &KubernetesDeploymentRouter{
 		kubeClient:    mocks.kubeClient,
 		flaggerClient: mocks.flaggerClient,

pkg/router/router_test.go

@@ -3,7 +3,6 @@ package router
 import (
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
-	hpav1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -19,7 +18,7 @@ import (
 	"github.com/weaveworks/flagger/pkg/logger"
 )
 
-type fakeClients struct {
+type fixture struct {
 	canary        *flaggerv1.Canary
 	abtest        *flaggerv1.Canary
 	appmeshCanary *flaggerv1.Canary
@@ -30,7 +29,7 @@ type fakeClients struct {
 	logger        *zap.SugaredLogger
 }
 
-func setupfakeClients() fakeClients {
+func newFixture() fixture {
 	canary := newMockCanary()
 	abtest := newMockABTest()
 	appmeshCanary := newMockCanaryAppMesh()
@@ -42,7 +41,7 @@ func setupfakeClients() fakeClients {
 	meshClient := fakeFlagger.NewSimpleClientset()
 	logger, _ := logger.NewLogger("debug")
 
-	return fakeClients{
+	return fixture{
 		canary:        canary,
 		abtest:        abtest,
 		appmeshCanary: appmeshCanary,
@@ -62,7 +61,7 @@ func newMockCanaryAppMesh() *flaggerv1.Canary {
 			Name:      "appmesh",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
@@ -108,7 +107,7 @@ func newMockCanary() *flaggerv1.Canary {
 			Name:      "podinfo",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
@@ -173,7 +172,7 @@ func newMockABTest() *flaggerv1.Canary {
 			Name:      "abtest",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "abtest",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
@@ -309,12 +308,12 @@ func newMockCanaryIngress() *flaggerv1.Canary {
 			Name:      "nginx",
 		},
 		Spec: flaggerv1.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
+			TargetRef: flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 			},
-			IngressRef: &hpav1.CrossVersionObjectReference{
+			IngressRef: &flaggerv1.CrossNamespaceObjectReference{
 				Name:       "podinfo",
 				APIVersion: "extensions/v1beta1",
 				Kind:       "Ingress",