Merge pull request #86 from cloudflare/refactor-status

Refactor alert status attrs
This commit is contained in:
Łukasz Mierzwa
2017-05-02 10:28:07 +01:00
committed by GitHub
22 changed files with 621 additions and 129 deletions

View File

@@ -2,7 +2,7 @@ NAME := unsee
VERSION := $(shell git describe --tags --always --dirty='-dev')
# Alertmanager instance used when running locally, points to mock data
MOCK_PATH := $(CURDIR)/mock/0.5
MOCK_PATH := $(CURDIR)/mock/0.6.1
ALERTMANAGER_URI := file://$(MOCK_PATH)
# Listen port when running locally
PORT := 8080

View File

@@ -12,7 +12,7 @@ import (
httpmock "gopkg.in/jarcoal/httpmock.v1"
)
var testVersions = []string{"0.4", "0.5"}
var testVersions = []string{"0.4", "0.5", "0.6.1"}
func TestGetAlerts(t *testing.T) {
log.SetLevel(log.ErrorLevel)

View File

@@ -50,7 +50,7 @@
<script type="application/json" id="alert-group-elements">
<% var cls_indicator = 'incident-indicator-danger' %>
<% if (alert.silenced || alert.inhibited) { cls_indicator = 'incident-indicator-success' } %>
<% if (alert.status === "suppressed") { cls_indicator = 'incident-indicator-success' } %>
<div>
<% if (alert.generatorURL) { %>
<a class="label label-list label-default"
@@ -74,12 +74,12 @@
<%- text %>
</a>
<% }) %>
<% if (alert.inhibited) { %>
<% if (alert.inhibitedBy.length) { %>
<%= Templates.Render('buttonLabel', {elem: 'span', elemClass: 'label label-list label-success', label: {key: '@inhibited', value: 'true', text: '@inhibited: true'}}) %>
<% } else { %>
<%= Templates.Render('buttonLabel', {elem: 'span', elemClass: 'label label-list label-danger', label: {key: '@inhibited', value: 'false', text: '@inhibited: false'}}) %>
<% } %>
<% if (alert.silenced) { %>
<% if (alert.silencedBy.length) { %>
<%= Templates.Render('buttonLabel', {elem: 'span', elemClass: 'label label-list label-success', label: {key: '@silenced', value: 'true', text: '@silenced: true'}}) %>
<% } else { %>
<%= Templates.Render('buttonLabel', {elem: 'span', elemClass: 'label label-list label-danger', label: {key: '@silenced', value: 'false', text: '@silenced: false'}}) %>
@@ -110,28 +110,30 @@
</script>
<script type="application/json" id="alert-group-silence">
<div>
<% var silence = silences[alert.silenced] %>
<% if (silence) { %>
<blockquote class="silence-comment">
<% if (silence.jiraURL) { %>
<a href="<%= silence.jiraURL %>" target="_blank">
<i class="fa fa-external-link"/>
<% _.each(alert.silencedBy, function(silenceID) { %>
<div>
<% var silence = silences[silenceID] %>
<% if (silence) { %>
<blockquote class="silence-comment">
<% if (silence.jiraURL) { %>
<a href="<%= silence.jiraURL %>" target="_blank">
<i class="fa fa-external-link"/>
<%- silence.comment %>
</a>
<% } else { %>
<%- silence.comment %>
</a>
<% } else { %>
<%- silence.comment %>
<% } %>
<footer>
<cite>
<abbr class="label-ts" data-toggle="tooltip" data-placement="top" data-ts="<%= silence.startsAt %>">
<%- silence.createdBy %>
</abbr>
</cite>
</footer>
</blockquote>
<% } %>
</div>
<% } %>
<footer>
<cite>
<abbr class="label-ts" data-toggle="tooltip" data-placement="top" data-ts="<%= silence.startsAt %>">
<%- silence.createdBy %>
</abbr>
</cite>
</footer>
</blockquote>
<% } %>
</div>
<% }) %>
</script>
<script type="application/json" id="alert-group-label-map">
@@ -217,7 +219,7 @@
<% }) %>
<% var silencedText = '@silenced: false' %>
<% var isSilenced = 'false' %>
<% if (alert.silenced) { %>
<% if (alert.silencedBy.length) { %>
<% silencedText = '@silenced: true' %>
<% isSilenced = 'true' %>
<% } %>
@@ -226,7 +228,7 @@
<% var inhibitedText = '@inhibited: false' %>
<% var isInhibited = 'false' %>
<% if (alert.inhibited) { %>
<% if (alert.inhibitedBy.length) { %>
<% inhibitedText = '@inhibited: true' %>
<% isInhibited = 'true' %>
<% } %>
@@ -241,7 +243,7 @@
<%= Templates.Render('alertGroupAnnotations', {alert: alert}) %>
<%= Templates.Render('alertGroupLabels', {alert: alert, group: group}) %>
<%= Templates.Render('alertGroupElements', {alert: alert}) %>
<% if (alert.silenced) { %>
<% if (alert.silencedBy.length) { %>
<%= Templates.Render('alertGroupSilence', {alert: alert, silences: silences}) %>
<% } %>
</div>

File diff suppressed because one or more lines are too long

View File

@@ -41,8 +41,8 @@ func (filter *fuzzyFilter) Match(alert *models.Alert, matches int) bool {
}
}
if alert.Silenced != "" {
if silence := store.Store.GetSilence(alert.Silenced); silence != nil {
for _, silenceID := range alert.SilencedBy {
if silence := store.Store.GetSilence(silenceID); silence != nil {
if filter.Matcher.Compare(silence.Comment, filter.Value) {
filter.Hits++
return true

View File

@@ -30,7 +30,7 @@ func (filter *inhibitedFilter) init(name string, matcher *matcherT, rawText stri
func (filter *inhibitedFilter) Match(alert *models.Alert, matches int) bool {
if filter.IsValid {
isMatch := filter.Matcher.Compare(alert.Inhibited, filter.Value)
isMatch := filter.Matcher.Compare(alert.IsInhibited(), filter.Value)
if isMatch {
filter.Hits++
}

View File

@@ -15,9 +15,11 @@ type silenceAuthorFilter struct {
func (filter *silenceAuthorFilter) Match(alert *models.Alert, matches int) bool {
if filter.IsValid {
var isMatch bool
if alert.Silenced != "" {
if silence := store.Store.GetSilence(alert.Silenced); silence != nil {
isMatch = filter.Matcher.Compare(filter.Value, silence.CreatedBy)
if alert.IsSilenced() {
for _, silenceID := range alert.SilencedBy {
if silence := store.Store.GetSilence(silenceID); silence != nil {
isMatch = filter.Matcher.Compare(filter.Value, silence.CreatedBy)
}
}
} else {
isMatch = filter.Matcher.Compare("", filter.Value)
@@ -39,16 +41,18 @@ func newSilenceAuthorFilter() FilterT {
func sinceAuthorAutocomplete(name string, operators []string, alerts []models.Alert) []models.Autocomplete {
tokens := map[string]models.Autocomplete{}
for _, alert := range alerts {
if alert.Silenced != "" {
if silence := store.Store.GetSilence(alert.Silenced); silence != nil {
for _, operator := range operators {
token := fmt.Sprintf("%s%s%s", name, operator, silence.CreatedBy)
tokens[token] = makeAC(token, []string{
name,
strings.TrimPrefix(name, "@"),
fmt.Sprintf("%s%s", name, operator),
silence.CreatedBy,
})
if alert.IsSilenced() {
for _, silenceID := range alert.SilencedBy {
if silence := store.Store.GetSilence(silenceID); silence != nil {
for _, operator := range operators {
token := fmt.Sprintf("%s%s%s", name, operator, silence.CreatedBy)
tokens[token] = makeAC(token, []string{
name,
strings.TrimPrefix(name, "@"),
fmt.Sprintf("%s%s", name, operator),
silence.CreatedBy,
})
}
}
}
}

View File

@@ -15,9 +15,11 @@ type silenceJiraFilter struct {
func (filter *silenceJiraFilter) Match(alert *models.Alert, matches int) bool {
if filter.IsValid {
var isMatch bool
if alert.Silenced != "" {
if silence := store.Store.GetSilence(alert.Silenced); silence != nil {
isMatch = filter.Matcher.Compare(silence.JiraID, filter.Value)
if alert.IsSilenced() {
for _, silenceID := range alert.SilencedBy {
if silence := store.Store.GetSilence(silenceID); silence != nil {
isMatch = filter.Matcher.Compare(silence.JiraID, filter.Value)
}
}
} else {
isMatch = filter.Matcher.Compare("", filter.Value)
@@ -39,17 +41,19 @@ func newSilenceJiraFilter() FilterT {
func sinceJiraIDAutocomplete(name string, operators []string, alerts []models.Alert) []models.Autocomplete {
tokens := map[string]models.Autocomplete{}
for _, alert := range alerts {
if alert.Silenced != "" {
silence := store.Store.GetSilence(alert.Silenced)
if silence != nil && silence.JiraID != "" {
for _, operator := range operators {
token := fmt.Sprintf("%s%s%s", name, operator, silence.JiraID)
tokens[token] = makeAC(token, []string{
name,
strings.TrimPrefix(name, "@"),
fmt.Sprintf("%s%s", name, operator),
silence.JiraID,
})
if alert.IsSilenced() {
for _, silenceID := range alert.SilencedBy {
silence := store.Store.GetSilence(silenceID)
if silence != nil && silence.JiraID != "" {
for _, operator := range operators {
token := fmt.Sprintf("%s%s%s", name, operator, silence.JiraID)
tokens[token] = makeAC(token, []string{
name,
strings.TrimPrefix(name, "@"),
fmt.Sprintf("%s%s", name, operator),
silence.JiraID,
})
}
}
}
}

View File

@@ -30,9 +30,7 @@ func (filter *silencedFilter) init(name string, matcher *matcherT, rawText strin
func (filter *silencedFilter) Match(alert *models.Alert, matches int) bool {
if filter.IsValid {
var isSilenced bool
isSilenced = (alert.Silenced != "")
isMatch := filter.Matcher.Compare(isSilenced, filter.Value)
isMatch := filter.Matcher.Compare(alert.IsSilenced(), filter.Value)
if isMatch {
filter.Hits++
}

View File

@@ -34,13 +34,13 @@ var tests = []filterTest{
filterTest{
Expression: "@silenced=true",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
IsMatch: true,
},
filterTest{
Expression: "@silenced!=true",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
IsMatch: false,
},
filterTest{
@@ -79,19 +79,19 @@ var tests = []filterTest{
filterTest{
Expression: "@inhibited=true",
IsValid: true,
Alert: models.Alert{Inhibited: true},
Alert: models.Alert{Status: "suppressed", InhibitedBy: []string{"999"}},
IsMatch: true,
},
filterTest{
Expression: "@inhibited=true",
IsValid: true,
Alert: models.Alert{Inhibited: false},
Alert: models.Alert{Status: "active"},
IsMatch: false,
},
filterTest{
Expression: "@inhibited!=true",
IsValid: true,
Alert: models.Alert{Inhibited: true},
Alert: models.Alert{Status: "suppressed", InhibitedBy: []string{"999"}},
IsMatch: false,
},
filterTest{
@@ -118,70 +118,70 @@ var tests = []filterTest{
filterTest{
Expression: "@silence_jira=1",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "1"},
IsMatch: true,
},
filterTest{
Expression: "@silence_jira=2",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1"},
IsMatch: false,
},
filterTest{
Expression: "@silence_jira!=3",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "x"},
IsMatch: true,
},
filterTest{
Expression: "@silence_jira!=4",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "4"},
IsMatch: false,
},
filterTest{
Expression: "@silence_jira!=5",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1"},
IsMatch: true,
},
filterTest{
Expression: "@silence_jira=~abc",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "xxabcxx"},
IsMatch: true,
},
filterTest{
Expression: "@silence_jira=~abc",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "xxx"},
IsMatch: false,
},
filterTest{
Expression: "@silence_jira=~",
IsValid: false,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "xxx"},
IsMatch: false,
},
filterTest{
Expression: "@silence_jira~=",
IsValid: false,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "xxx"},
IsMatch: false,
},
filterTest{
Expression: "@silence_jira~=1",
IsValid: false,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", JiraID: "xxx"},
IsMatch: false,
},
@@ -189,56 +189,56 @@ var tests = []filterTest{
filterTest{
Expression: "@silence_author=john",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", CreatedBy: "john"},
IsMatch: true,
},
filterTest{
Expression: "@silence_author=john",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", CreatedBy: "bob"},
IsMatch: false,
},
filterTest{
Expression: "@silence_author!=john",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", CreatedBy: "bob"},
IsMatch: true,
},
filterTest{
Expression: "@silence_author!=john",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", CreatedBy: "john"},
IsMatch: false,
},
filterTest{
Expression: "@silence_author!=john",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1"},
IsMatch: true,
},
filterTest{
Expression: "@silence_author=~",
IsValid: false,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1"},
IsMatch: false,
},
filterTest{
Expression: "@silence_author===x",
IsValid: false,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1"},
IsMatch: false,
},
filterTest{
Expression: "@silence_author=!!xxx",
IsValid: false,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1"},
IsMatch: false,
},
@@ -404,28 +404,28 @@ var tests = []filterTest{
filterTest{
Expression: "abc",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", Comment: "abc"},
IsMatch: true,
},
filterTest{
Expression: "abc",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", Comment: "abcxxx"},
IsMatch: true,
},
filterTest{
Expression: "abc",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", Comment: "ABCD"},
IsMatch: true,
},
filterTest{
Expression: "abc",
IsValid: true,
Alert: models.Alert{Silenced: "1"},
Alert: models.Alert{Status: "suppressed", SilencedBy: []string{"1"}},
Silence: models.Silence{ID: "1", Comment: "xzc"},
IsMatch: false,
},

View File

@@ -45,7 +45,7 @@ var (
Name: "unsee_collected_alerts",
Help: "Total number of alerts collected from Alertmanager API",
},
[]string{"silenced", "inhibited"},
[]string{"status"},
)
metricAlertGroups = prometheus.NewGauge(
prometheus.GaugeOpts{

View File

@@ -6,6 +6,7 @@ package v04
import (
"errors"
"strconv"
"time"
"github.com/blang/semver"
@@ -75,16 +76,26 @@ func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
alertList := models.AlertList{}
for _, b := range g.Blocks {
for _, a := range b.Alerts {
status := models.AlertStateActive
silencedBy := []string{}
if a.Silenced > 0 {
silencedBy = append(silencedBy, strconv.Itoa(a.Silenced))
status = models.AlertStateSuppressed
}
inhibitedBy := []string{}
if a.Inhibited {
inhibitedBy = append(inhibitedBy, "0")
status = models.AlertStateSuppressed
}
us := models.Alert{
Annotations: a.Annotations,
Labels: a.Labels,
StartsAt: a.StartsAt,
EndsAt: a.EndsAt,
GeneratorURL: a.GeneratorURL,
Inhibited: a.Inhibited,
}
if a.Silenced > 0 {
us.Silenced = string(a.Silenced)
Status: status,
InhibitedBy: inhibitedBy,
SilencedBy: silencedBy,
}
alertList = append(alertList, us)
}

View File

@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"math"
"strconv"
"time"
"github.com/blang/semver"
@@ -77,7 +78,7 @@ func (m SilenceMapper) GetSilences() ([]models.Silence, error) {
for _, s := range resp.Data.Silences {
us := models.Silence{
ID: string(s.ID),
ID: strconv.Itoa(s.ID),
Matchers: s.Matchers,
StartsAt: s.StartsAt,
EndsAt: s.EndsAt,

View File

@@ -73,14 +73,26 @@ func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
alertList := models.AlertList{}
for _, b := range g.Blocks {
for _, a := range b.Alerts {
status := models.AlertStateActive
silencedBy := []string{}
if a.Silenced != "" {
silencedBy = append(silencedBy, a.Silenced)
status = models.AlertStateSuppressed
}
inhibitedBy := []string{}
if a.Inhibited {
inhibitedBy = append(inhibitedBy, "0")
status = models.AlertStateSuppressed
}
us := models.Alert{
Annotations: a.Annotations,
Labels: a.Labels,
StartsAt: a.StartsAt,
EndsAt: a.EndsAt,
GeneratorURL: a.GeneratorURL,
Inhibited: a.Inhibited,
Silenced: a.Silenced,
Status: status,
InhibitedBy: inhibitedBy,
SilencedBy: silencedBy,
}
alertList = append(alertList, us)
}

View File

@@ -75,9 +75,13 @@ func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
alertList := models.AlertList{}
for _, b := range g.Blocks {
for _, a := range b.Alerts {
silenceID := ""
if len(a.SilencedBy) > 0 {
silenceID = a.SilencedBy[0]
inhibitedBy := []string{}
if a.InhibitedBy != nil {
inhibitedBy = a.InhibitedBy
}
silencedBy := []string{}
if a.SilencedBy != nil {
silencedBy = a.SilencedBy
}
us := models.Alert{
Annotations: a.Annotations,
@@ -85,8 +89,9 @@ func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
StartsAt: a.StartsAt,
EndsAt: a.EndsAt,
GeneratorURL: a.GeneratorURL,
Inhibited: len(a.InhibitedBy) > 0,
Silenced: silenceID,
Status: a.Status,
InhibitedBy: inhibitedBy,
SilencedBy: silencedBy,
}
alertList = append(alertList, us)
}

View File

@@ -0,0 +1,317 @@
{
"status": "success",
"data": [
{
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging"
},
"groupKey": "{}:{alertname=\"Free_Disk_Space_Too_Low\", cluster=\"staging\"}",
"blocks": [
{
"routeOpts": {
"receiver": "default",
"groupBy": [
"alertname",
"cluster",
"service"
],
"groupWait": 30000000000,
"groupInterval": 300000000000,
"repeatInterval": 10800000000000
},
"alerts": [
{
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
}
]
}
]
},
{
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev"
},
"groupKey": "{}:{alertname=\"HTTP_Probe_Failed\", cluster=\"dev\"}",
"blocks": [
{
"routeOpts": {
"receiver": "default",
"groupBy": [
"alertname",
"cluster",
"service"
],
"groupWait": 30000000000,
"groupInterval": 300000000000,
"repeatInterval": 10800000000000
},
"alerts": [
{
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
},
{
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "suppressed",
"inhibitedBy": [],
"silencedBy": [
"419caf4f-c9d3-4a73-bb78-cfef2bc703e9"
]
}
]
}
]
},
{
"labels": {
"alertname": "Host_Down"
},
"groupKey": "{}:{alertname=\"Host_Down\"}",
"blocks": [
{
"routeOpts": {
"receiver": "default",
"groupBy": [
"alertname",
"cluster",
"service"
],
"groupWait": 30000000000,
"groupInterval": 300000000000,
"repeatInterval": 10800000000000
},
"alerts": [
{
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "suppressed",
"inhibitedBy": [],
"silencedBy": [
"337b3ba9-aa95-4562-924b-be95335bafae"
]
},
{
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "suppressed",
"inhibitedBy": [],
"silencedBy": [
"337b3ba9-aa95-4562-924b-be95335bafae"
]
},
{
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "suppressed",
"inhibitedBy": [],
"silencedBy": [
"337b3ba9-aa95-4562-924b-be95335bafae"
]
},
{
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
},
{
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
},
{
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
},
{
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
},
{
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"annotations": {
"summary": "Example summary"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
}
]
}
]
},
{
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod"
},
"groupKey": "{}:{alertname=\"Memory_Usage_Too_High\", cluster=\"prod\"}",
"blocks": [
{
"routeOpts": {
"receiver": "default",
"groupBy": [
"alertname",
"cluster",
"service"
],
"groupWait": 30000000000,
"groupInterval": 300000000000,
"repeatInterval": 10800000000000
},
"alerts": [
{
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"startsAt": "2017-05-01T20:34:21.20891774+01:00",
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"Status": "active",
"inhibitedBy": [],
"silencedBy": []
}
]
}
]
}
]
}

View File

@@ -0,0 +1,40 @@
{
"status": "success",
"data": [
{
"id": "419caf4f-c9d3-4a73-bb78-cfef2bc703e9",
"matchers": [
{
"name": "instance",
"value": "web1",
"isRegex": false
}
],
"startsAt": "2017-05-01T19:34:21.205359295Z",
"endsAt": "2063-01-01T00:00:00Z",
"updatedAt": "2017-05-01T19:34:21.205359295Z",
"createdBy": "john@example.com",
"comment": "Silenced instance"
},
{
"id": "337b3ba9-aa95-4562-924b-be95335bafae",
"matchers": [
{
"name": "alertname",
"value": "Host_Down",
"isRegex": false
},
{
"name": "cluster",
"value": "dev",
"isRegex": false
}
],
"startsAt": "2017-05-01T19:34:21.207248344Z",
"endsAt": "2063-01-01T00:00:00Z",
"updatedAt": "2017-05-01T19:34:21.207248344Z",
"createdBy": "john@example.com",
"comment": "Silenced Host_Down alerts in the dev cluster"
}
]
}

32
mock/0.6.1/api/v1/status Normal file
View File

@@ -0,0 +1,32 @@
{
"status": "success",
"data": {
"configJSON": {
"global": {},
"route": {},
"inhibit_rules": [],
"receivers": [],
"templates": []
},
"versionInfo": {
"branch": "HEAD",
"buildDate": "20170501-19:31:12",
"buildUser": "lukasz@localhost",
"goVersion": "go1.8.1",
"revision": "5aeaf2cb988b0094b7998bbe6568bba23a0eb9c2",
"version": "0.6.1"
},
"uptime": "2017-05-01T20:31:38.102687341+01:00",
"meshStatus": {
"name": "28:d2:44:df:95:eb",
"nickName": "localhost",
"peers": [
{
"name": "28:d2:44:df:95:eb",
"nickName": "localhost",
"uid": 18388267050726621000
}
]
}
}
}

View File

@@ -23,6 +23,23 @@ type Silence struct {
JiraURL string `json:"jiraURL"`
}
// AlertStateUnprocessed means that Alertmanager notify didn't yet process it
// and AM doesn't know if alert is active or suppressed
const AlertStateUnprocessed = "unprocessed"
// AlertStateActive is the state in which we know that the alert should fire
const AlertStateActive = "active"
// AlertStateSuppressed means that we know that alert is silenced or inhibited
const AlertStateSuppressed = "suppressed"
// AlertStateList exports all alert states so other packages can get this list
var AlertStateList = []string{
AlertStateUnprocessed,
AlertStateActive,
AlertStateSuppressed,
}
// Alert is vanilla alert + some additional attributes
// unsee extends an alert object with:
// * Links map, it's generated from annotations if annotation value is an url
@@ -35,13 +52,29 @@ type Alert struct {
StartsAt time.Time `json:"startsAt"`
EndsAt time.Time `json:"endsAt"`
GeneratorURL string `json:"generatorURL"`
Inhibited bool `json:"inhibited"`
Silenced string `json:"silenced"`
Status string `json:"status"`
SilencedBy []string `json:"silencedBy"`
InhibitedBy []string `json:"inhibitedBy"`
// unsee fields
Links map[string]string `json:"links"`
Fingerprint string `json:"-"`
}
// IsSilenced will return true if alert should be considered silenced
func (a Alert) IsSilenced() bool {
return (a.Status == AlertStateSuppressed && len(a.SilencedBy) > 0)
}
// IsInhibited will return true if alert should be considered silenced
func (a Alert) IsInhibited() bool {
return (a.Status == AlertStateSuppressed && len(a.InhibitedBy) > 0)
}
// IsActive will return true if alert is not suppressed in any way
func (a Alert) IsActive() bool {
return (a.Status == AlertStateActive)
}
// AlertList is flat list of UnseeAlert objects
type AlertList []Alert

View File

@@ -60,10 +60,9 @@ func PullFromAlertmanager() {
acMap := map[string]models.Autocomplete{}
metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "true"}).Set(0)
metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "false"}).Set(0)
metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "true"}).Set(0)
metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "false"}).Set(0)
for _, state := range models.AlertStateList {
metricAlerts.With(prometheus.Labels{"status": state}).Set(0)
}
for _, ag := range alertGroups {
// used to generate group content hash
@@ -100,19 +99,7 @@ func PullFromAlertmanager() {
ag.Alerts = []models.Alert{}
for _, alert := range alerts {
ag.Alerts = append(ag.Alerts, alert)
if alert.Silenced != "" {
if alert.Inhibited {
metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "true"}).Inc()
} else {
metricAlerts.With(prometheus.Labels{"silenced": "true", "inhibited": "false"}).Inc()
}
} else {
if alert.Inhibited {
metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "true"}).Inc()
} else {
metricAlerts.With(prometheus.Labels{"silenced": "false", "inhibited": "false"}).Inc()
}
}
metricAlerts.With(prometheus.Labels{"status": alert.Status}).Inc()
}
for _, hint := range transform.BuildAutocomplete(ag.Alerts) {

View File

@@ -162,18 +162,20 @@ func alerts(c *gin.Context) {
}
io.WriteString(h, string(aj))
if alert.Silenced != "" {
if silence := store.Store.GetSilence(alert.Silenced); silence != nil {
silences[alert.Silenced] = *silence
}
if alert.IsSilenced() {
countLabel(counters, "@silenced", "true")
for _, silenceID := range alert.SilencedBy {
if silence := store.Store.GetSilence(silenceID); silence != nil {
silences[silenceID] = *silence
}
}
} else {
countLabel(counters, "@silenced", "false")
}
countLabel(counters, "@inhibited", strconv.FormatBool(alert.Inhibited))
countLabel(counters, "@inhibited", strconv.FormatBool(alert.IsInhibited()))
if alert.Silenced == "" && !alert.Inhibited {
if alert.IsActive() {
agCopy.ActiveCount++
}

View File

@@ -20,7 +20,16 @@ import (
"gopkg.in/jarcoal/httpmock.v1"
)
var testVersions = []string{"0.4", "0.5"}
var testVersions = []string{"0.4", "0.5", "0.6.1"}
func stringInSlice(stringArray []string, value string) bool {
for _, s := range stringArray {
if s == value {
return true
}
}
return false
}
func mockConfig() {
log.SetLevel(log.ErrorLevel)
@@ -150,6 +159,41 @@ func TestAlerts(t *testing.T) {
if len(a.Links) != 1 {
t.Errorf("Invalid number of links, got %d, expected 1, %v", len(a.Links), a)
}
if a.InhibitedBy == nil {
t.Errorf("InhibitedBy is nil, %v", a)
}
if a.SilencedBy == nil {
t.Errorf("SilencedBy is nil, %v", a)
}
}
}
}
}
func TestValidateAllAlerts(t *testing.T) {
mockConfig()
for _, version := range testVersions {
mockAlerts(version)
r := ginTestEngine()
req, _ := http.NewRequest("GET", "/alerts.json?q=alertname=HTTP_Probe_Failed,instance=web1", nil)
resp := httptest.NewRecorder()
r.ServeHTTP(resp, req)
if resp.Code != http.StatusOK {
t.Errorf("GET /alerts.json returned status %d", resp.Code)
}
ur := models.AlertsResponse{}
json.Unmarshal(resp.Body.Bytes(), &ur)
for _, ag := range ur.AlertGroups {
for _, a := range ag.Alerts {
if !stringInSlice(models.AlertStateList, a.Status) {
t.Errorf("Invalid alert status '%s', not in %v", a.Status, models.AlertStateList)
}
if a.InhibitedBy == nil {
t.Errorf("InhibitedBy is nil, %v", a)
}
if a.SilencedBy == nil {
t.Errorf("SilencedBy is nil, %v", a)
}
}
}
}