Merge pull request #56 from cloudflare/inhibited

Allow filtering inhibited alerts

Makefile

@@ -48,6 +48,7 @@ run: $(NAME)
 	COLOR_LABELS_UNIQUE="instance cluster" \
 	COLOR_LABELS_STATIC="job" \
 	DEBUG="$(GIN_DEBUG)" \
+	FILTER_DEFAULT="@inhibited=false" \
 	PORT=$(PORT) \
 	./$(NAME)
 

assets/static/autocomplete.js

@@ -34,6 +34,10 @@ var Autocomplete = (function() {
         // static list of hints for @silenced label
         hints.push('@silenced=true');
         hints.push('@silenced=false');
+      } else if (label_key == '@inhibited') {
+        // static list of hints for @inhibited label
+        hints.push('@inhibited=true');
+        hints.push('@inhibited=false');
       } else {
         // equal and non-equal hints for everything else
         hints.push(label_key + '=' + label_val);

assets/static/colors.js

@@ -6,7 +6,9 @@ var Colors = (function() {
 
     var specialLabels = {
         '@silenced: false': 'label-danger',
-        '@silenced: true': 'label-success'
+        '@silenced: true': 'label-success',
+        '@inhibited: false': 'label-danger',
+        '@inhibited: true': 'label-success'
     }
 
 

assets/templates/alertgroup.html

@@ -50,7 +50,7 @@
 
 <script type="application/json" id="alert-group-elements">
   <% var cls_indicator = 'incident-indicator-danger' %>
-  <% if (alert.silenced) { cls_indicator = 'incident-indicator-success' } %>
+  <% if (alert.silenced || alert.inhibited) { cls_indicator = 'incident-indicator-success' } %>
   <div>
     <% if (alert.generatorURL) { %>
       <a class="label label-list label-default"
@@ -74,6 +74,11 @@
         <%- text %>
       </a>
     <% }) %>
+    <% if (alert.inhibited) { %>
+      <%= Templates.Render('buttonLabel', {elem: 'span', elemClass: 'label label-list label-success', label: {key: '@inhibited', value: 'true', text: '@inhibited: true'}}) %>
+    <% } else { %>
+      <%= Templates.Render('buttonLabel', {elem: 'span', elemClass: 'label label-list label-danger', label: {key: '@inhibited', value: 'false', text: '@inhibited: false'}}) %>
+    <% } %>
     <% if (alert.silenced) { %>
       <%= Templates.Render('buttonLabel', {elem: 'span', elemClass: 'label label-list label-success', label: {key: '@silenced', value: 'true', text: '@silenced: true'}}) %>
     <% } else { %>
@@ -206,6 +211,16 @@
           <% } %>
           <% if (labelMap[silencedText] == undefined) { labelMap[silencedText] = {key: '@silenced', value: isSilenced, hits: 0} } %>
           <% labelMap[silencedText].hits++ %>
+
+          <% var inhibitedText = '@inhibited: false' %>
+          <% var isInhibited = 'false' %>
+          <% if (alert.inhibited) { %>
+            <% inhibitedText = '@inhibited: true' %>
+            <% isInhibited = 'true' %>
+          <% } %>
+          <% if (labelMap[inhibitedText] == undefined) { labelMap[inhibitedText] = {key: '@inhibited', value: isInhibited, hits: 0} } %>
+          <% labelMap[inhibitedText].hits++ %>
+
         <% } else { %>
           <% var cls_body = '' %>
           <% if (i < group.alerts.length - 1) { cls_body = 'incident-group-separator' } %>

assets/templates/help.html

@@ -149,6 +149,26 @@
                             </table>
                         </td>
                     </tr>
+                    <tr>
+                        <td id="help-inhibited">
+                            <code>@inhibited=(true false)</code>
+                        </td>
+                        <td>
+                            <p>Match alerts based on inhibition status.</p>
+                            <table class="table examples">
+                                <tbody>
+                                    <tr>
+                                        <td><span class="label label-info">@inhibited=true</span></td>
+                                        <td>Match only inhibited alerts.</td>
+                                    </tr>
+                                    <tr>
+                                        <td><span class="label label-info">@inhibited=false</span></td>
+                                        <td>Match only non-inhibited alerts.</td>
+                                    </tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
                     <tr>
                         <td id="help-silence_author">
                             <code>@silence_author(= != =~ !~)$value</code>

filters/filter_inhibited.go

@@ -0,0 +1,72 @@
+package filters
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/cloudflare/unsee/models"
+)
+
+type inhibitedFilter struct {
+	alertFilter
+}
+
+func (filter *inhibitedFilter) init(name string, matcher *matcherT, rawText string, isValid bool, value string) {
+	filter.Matched = name
+	if matcher != nil {
+		filter.Matcher = *matcher
+	}
+	filter.RawText = rawText
+	filter.IsValid = isValid
+	switch value {
+	case "true":
+		filter.Value = true
+	case "false":
+		filter.Value = false
+	default:
+		filter.IsValid = false
+	}
+}
+
+func (filter *inhibitedFilter) Match(alert *models.Alert, matches int) bool {
+	if filter.IsValid {
+		isMatch := filter.Matcher.Compare(alert.Inhibited, filter.Value)
+		if isMatch {
+			filter.Hits++
+		}
+		return isMatch
+	}
+	e := fmt.Sprintf("Match() called on invalid filter %#v", filter)
+	panic(e)
+}
+
+func newInhibitedFilter() FilterT {
+	f := inhibitedFilter{}
+	return &f
+}
+
+func inhibitedAutocomplete(name string, operators []string, alerts []models.Alert) []models.Autocomplete {
+	tokens := []models.Autocomplete{}
+	for _, operator := range operators {
+		switch operator {
+		case equalOperator:
+			tokens = append(tokens, makeAC(
+				fmt.Sprintf("%s%strue", name, operator),
+				[]string{
+					name,
+					strings.TrimPrefix(name, "@"),
+					fmt.Sprintf("%s%s", name, operator),
+				},
+			))
+			tokens = append(tokens, makeAC(
+				fmt.Sprintf("%s%sfalse", name, operator),
+				[]string{
+					name,
+					strings.TrimPrefix(name, "@"),
+					fmt.Sprintf("%s%s", name, operator),
+				},
+			))
+		}
+	}
+	return tokens
+}

filters/filter_test.go

@@ -48,6 +48,41 @@ var tests = []filterTest{
 		IsValid:    false,
 	},
 
+	filterTest{
+		Expression: "@inhibited=true",
+		IsValid:    true,
+		Alert:      models.Alert{},
+		IsMatch:    false,
+	},
+	filterTest{
+		Expression: "@inhibited!=true",
+		IsValid:    true,
+		Alert:      models.Alert{},
+		IsMatch:    true,
+	},
+	filterTest{
+		Expression: "@inhibited=true",
+		IsValid:    true,
+		Alert:      models.Alert{Inhibited: true},
+		IsMatch:    true,
+	},
+	filterTest{
+		Expression: "@inhibited=true",
+		IsValid:    true,
+		Alert:      models.Alert{Inhibited: false},
+		IsMatch:    false,
+	},
+	filterTest{
+		Expression: "@inhibited!=true",
+		IsValid:    true,
+		Alert:      models.Alert{Inhibited: true},
+		IsMatch:    false,
+	},
+	filterTest{
+		Expression: "@inhibited=xx",
+		IsValid:    false,
+	},
+
 	filterTest{
 		Expression: "@silence_jira=1",
 		IsValid:    true,

filters/registry.go

@@ -35,6 +35,12 @@ var AllFilters = []filterConfig{
 		Factory:            newSilencedFilter,
 		Autocomplete:       silencedAutocomplete,
 	},
+	filterConfig{
+		Label:              "@inhibited",
+		SupportedOperators: []string{equalOperator, notEqualOperator},
+		Factory:            newInhibitedFilter,
+		Autocomplete:       inhibitedAutocomplete,
+	},
 	filterConfig{
 		Label:              "@age",
 		SupportedOperators: []string{lessThanOperator, moreThanOperator},

main.go

@@ -45,7 +45,7 @@ var (
 			Name: "unsee_collected_alerts",
 			Help: "Total number of alerts collected from Alertmanager API",
 		},
-		[]string{"silenced"},
+		[]string{"silenced", "inhibited"},
 	)
 	metricAlertGroups = prometheus.NewGauge(
 		prometheus.GaugeOpts{

mock/0.4/api/v1/alerts/groups

@@ -272,7 +272,7 @@
               "startsAt": "2017-02-18T01:14:38Z",
               "endsAt": "0001-01-01T00:00:00Z",
               "generatorURL": "http://localhost/graph",
-              "inhibited": false
+              "inhibited": true
             }
           ]
         }

mock/0.5/api/v1/alerts/groups

@@ -272,7 +272,7 @@
               "startsAt": "2017-02-18T01:14:38Z",
               "endsAt": "0001-01-01T00:00:00Z",
               "generatorURL": "http://localhost/graph",
-              "inhibited": false
+              "inhibited": true
             }
           ]
         }

timer.go

@@ -57,9 +57,10 @@ func PullFromAlertmanager() {
 
 	acMap := map[string]models.Autocomplete{}
 
-	// counters used to update metrics
-	var counterAlertsSilenced float64
-	var counterAlertsUnsilenced float64
+	metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "true"}).Set(0)
+	metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "false"}).Set(0)
+	metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "true"}).Set(0)
+	metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "false"}).Set(0)
 
 	for _, ag := range alertGroups {
 		// used to generate group content hash
@@ -97,9 +98,17 @@ func PullFromAlertmanager() {
 		for _, alert := range alerts {
 			ag.Alerts = append(ag.Alerts, alert)
 			if alert.Silenced != "" {
-				counterAlertsSilenced++
+				if alert.Inhibited {
+					metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "true"}).Inc()
+				} else {
+					metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "false"}).Inc()
+				}
 			} else {
-				counterAlertsUnsilenced++
+				if alert.Inhibited {
+					metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "true"}).Inc()
+				} else {
+					metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "false"}).Inc()
+				}
 			}
 		}
 
@@ -125,8 +134,6 @@ func PullFromAlertmanager() {
 	alertManagerError = ""
 	errorLock.Unlock()
 
-	metricAlerts.With(prometheus.Labels{"silenced": "true"}).Set(counterAlertsSilenced)
-	metricAlerts.With(prometheus.Labels{"silenced": "false"}).Set(counterAlertsUnsilenced)
 	metricAlertGroups.Set(float64(len(alertStore)))
 
 	store.Store.Update(alertStore, colorStore, acStore)

views.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -165,6 +166,8 @@ func alerts(c *gin.Context) {
 					countLabel(counters, "@silenced", "false")
 				}
 
+				countLabel(counters, "@inhibited", strconv.FormatBool(alert.Inhibited))
+
 				for key, value := range alert.Labels {
 					if keyMap, foundKey := store.Store.Colors[key]; foundKey {
 						if color, foundColor := keyMap[value]; foundColor {

views_test.go

@@ -142,7 +142,7 @@ func TestAlerts(t *testing.T) {
 		if ur.Status != "success" {
 			t.Errorf("[%s] Invalid status in response: %s", version, ur.Status)
 		}
-		if len(ur.Counters) != 5 {
+		if len(ur.Counters) != 6 {
 			t.Errorf("[%s] Invalid number of counters in response (%d): %v", version, len(ur.Counters), ur.Counters)
 		}
 		for _, ag := range ur.AlertGroups {
@@ -238,6 +238,8 @@ var acTests = []acTestCase{
 			"@age<1h",
 			"@age>10m",
 			"@age>1h",
+			"@inhibited=false",
+			"@inhibited=true",
 			"@limit=10",
 			"@limit=50",
 			"@silence_author!=john@example.com",