From 26d14d1bd2b847aae7827d7ca25b96675d9a4f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Sat, 24 Jun 2017 20:37:20 -0700 Subject: [PATCH] Refactor Alertmanager API client code to use multiple upstream instances Alerts are stored per instance and deduplicated on read. --- CONTRIBUTING.md | 2 +- Makefile | 2 +- alertmanager/alertmanager.go | 62 ------- alertmanager/alertmanager_test.go | 66 ------- alertmanager/dedup.go | 111 ++++++++++++ alertmanager/mapper.go | 19 ++ alertmanager/models.go | 281 ++++++++++++++++++++++++++++++ alertmanager/slices.go | 19 ++ alertmanager/upstream.go | 250 +++----------------------- alertmanager/version.go | 8 +- assets/static/alerts.js | 7 +- assets/templates/alertgroup.html | 16 +- bindata_assetfs.go | 4 +- config/config.go | 2 +- filters/filter_fuzzy.go | 13 +- filters/filter_silence_author.go | 32 ++-- filters/filter_silence_jira.go | 33 ++-- filters/filter_test.go | 29 ++- main.go | 57 ++---- mapper/mapper.go | 5 +- mapper/v04/alerts.go | 7 +- mapper/v04/silences.go | 7 +- mapper/v05/alerts.go | 7 +- mapper/v05/silences.go | 7 +- mapper/v061/alerts.go | 7 +- mapper/v062/alerts.go | 7 +- models/models.go | 50 ++++-- store/store.go | 44 ----- store/store_test.go | 57 ------ timer.go | 142 ++------------- views.go | 57 ++---- views_test.go | 18 +- 32 files changed, 658 insertions(+), 770 deletions(-) delete mode 100644 alertmanager/alertmanager.go delete mode 100644 alertmanager/alertmanager_test.go create mode 100644 alertmanager/dedup.go create mode 100644 alertmanager/mapper.go create mode 100644 alertmanager/models.go create mode 100644 alertmanager/slices.go delete mode 100644 store/store.go delete mode 100644 store/store_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4350dced4..3a9761c01 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,4 +83,4 @@ To support a new release that breaks API following changes needs to be done: silences (depending if both need a new code) under mapper/vXY (X major Alertmanager version, Y minor version). * Register new mapper in the `init()` function in the - `alertmanager/alertmanager.go` file. + `alertmanager/mapper.go` file. diff --git a/Makefile b/Makefile index 8b3d57c9c..5ce4e8048 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ VERSION := $(shell git describe --tags --always --dirty='-dev') # Alertmanager instance used when running locally, points to mock data MOCK_PATH := $(CURDIR)/mock/0.7.1 -ALERTMANAGER_URI := file://$(MOCK_PATH) +ALERTMANAGER_URI := "localfs:file://$(MOCK_PATH) local2:file://$(CURDIR)/mock/0.7.0" # Listen port when running locally PORT := 8080 diff --git a/alertmanager/alertmanager.go b/alertmanager/alertmanager.go deleted file mode 100644 index c48fe5723..000000000 --- a/alertmanager/alertmanager.go +++ /dev/null @@ -1,62 +0,0 @@ -package alertmanager - -import ( - "time" - - "github.com/cloudflare/unsee/mapper" - "github.com/cloudflare/unsee/mapper/v04" - "github.com/cloudflare/unsee/mapper/v05" - "github.com/cloudflare/unsee/mapper/v061" - "github.com/cloudflare/unsee/mapper/v062" - "github.com/cloudflare/unsee/models" - - log "github.com/Sirupsen/logrus" -) - -// initialize all mappers -func init() { - mapper.RegisterAlertMapper(v04.AlertMapper{}) - mapper.RegisterAlertMapper(v05.AlertMapper{}) - mapper.RegisterAlertMapper(v061.AlertMapper{}) - mapper.RegisterAlertMapper(v062.AlertMapper{}) - mapper.RegisterSilenceMapper(v04.SilenceMapper{}) - mapper.RegisterSilenceMapper(v05.SilenceMapper{}) -} - -// GetAlerts will send request to Alertmanager and return list of alert groups -// from the API -func GetAlerts(version string) ([]models.AlertGroup, error) { - groups := []models.AlertGroup{} - - mapper, err := mapper.GetAlertMapper(version) - if err != nil { - return groups, err - } - - start := time.Now() - groups, err = mapper.GetAlerts() - if err != nil { - return groups, err - } - log.Infof("Got %d alert group(s) in %s", len(groups), time.Since(start)) - return groups, nil -} - -// GetSilences will send request to Alertmanager and return list of silences -// from the API -func GetSilences(version string) ([]models.Silence, error) { - silences := []models.Silence{} - - mapper, err := mapper.GetSilenceMapper(version) - if err != nil { - return silences, err - } - - start := time.Now() - silences, err = mapper.GetSilences() - if err != nil { - return silences, err - } - log.Infof("Got %d silences(s) in %s", len(silences), time.Since(start)) - return silences, nil -} diff --git a/alertmanager/alertmanager_test.go b/alertmanager/alertmanager_test.go deleted file mode 100644 index 4381a5803..000000000 --- a/alertmanager/alertmanager_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package alertmanager_test - -import ( - "testing" - - "github.com/cloudflare/unsee/alertmanager" - "github.com/cloudflare/unsee/config" - "github.com/cloudflare/unsee/mock" - - log "github.com/Sirupsen/logrus" - httpmock "gopkg.in/jarcoal/httpmock.v1" -) - -func TestGetAlerts(t *testing.T) { - log.SetLevel(log.ErrorLevel) - config.Config.AlertmanagerURI = "http://localhost" - - httpmock.Activate() - defer httpmock.DeactivateAndReset() - - for _, version := range mock.ListAllMocks() { - httpmock.Reset() - mock.RegisterURL("http://localhost/api/v1/status", version, "status") - mock.RegisterURL("http://localhost/api/v1/alerts/groups", version, "alerts/groups") - - v := alertmanager.GetVersion() - if v != version { - t.Errorf("GetVersion() returned '%s', expected '%s'", v, version) - } - - groups, err := alertmanager.GetAlerts(v) - if err != nil { - t.Errorf("GetAlerts(%s) failed: %s", version, err.Error()) - } - if len(groups) != 10 { - t.Errorf("Got %d groups, expected 10", len(groups)) - } - } -} - -func TestGetSilences(t *testing.T) { - log.SetLevel(log.ErrorLevel) - config.Config.AlertmanagerURI = "http://localhost" - - httpmock.Activate() - defer httpmock.DeactivateAndReset() - - for _, version := range mock.ListAllMocks() { - httpmock.Reset() - mock.RegisterURL("http://localhost/api/v1/status", version, "status") - mock.RegisterURL("http://localhost/api/v1/silences", version, "silences") - - v := alertmanager.GetVersion() - if v != version { - t.Errorf("GetVersion() returned '%s', expected '%s'", v, version) - } - - silences, err := alertmanager.GetSilences(v) - if err != nil { - t.Errorf("GetSilences(%s) failed: %s", version, err.Error()) - } - if len(silences) != 3 { - t.Errorf("Got %d silences, expected %d", len(silences), 3) - } - } -} diff --git a/alertmanager/dedup.go b/alertmanager/dedup.go new file mode 100644 index 000000000..9d36e8137 --- /dev/null +++ b/alertmanager/dedup.go @@ -0,0 +1,111 @@ +package alertmanager + +import ( + "sort" + + "github.com/cloudflare/unsee/models" +) + +// DedupAlerts will collect alert groups from all defined Alertmanager +// upstreams and deduplicate them, so we only return unique alerts +func DedupAlerts() []models.AlertGroup { + uniqueGroups := map[string][]models.AlertGroup{} + + upstreams := GetAlertmanagers() + for _, am := range upstreams { + groups := am.Alerts() + for _, ag := range groups { + if _, found := uniqueGroups[ag.ID]; !found { + uniqueGroups[ag.ID] = []models.AlertGroup{} + } + uniqueGroups[ag.ID] = append(uniqueGroups[ag.ID], ag) + } + } + + dedupedGroups := []models.AlertGroup{} + for _, agList := range uniqueGroups { + alerts := map[string]models.Alert{} + for _, ag := range agList { + for _, alert := range ag.Alerts { + a, found := alerts[alert.ID] + if found { + for _, am := range alert.Alertmanager { + a.Alertmanager = append(a.Alertmanager, am) + alerts[alert.ID] = a + } + } else { + alerts[alert.ID] = alert + } + } + } + ag := models.AlertGroup(agList[0]) + ag.Alerts = models.AlertList{} + for _, alert := range alerts { + ag.Alerts = append(ag.Alerts, alert) + } + sort.Sort(ag.Alerts) + ag.Hash = ag.ContentFingerprint() + dedupedGroups = append(dedupedGroups, ag) + } + + return dedupedGroups +} + +// DedupColors returns a color map merged from all Alertmanager upstream color +// maps +func DedupColors() models.LabelsColorMap { + dedupedColors := models.LabelsColorMap{} + + upstreams := GetAlertmanagers() + + for _, am := range upstreams { + colors := am.Colors() + // map[string]map[string]LabelColors + for labelName, valueMap := range colors { + if _, found := dedupedColors[labelName]; !found { + dedupedColors[labelName] = map[string]models.LabelColors{} + } + for labelVal, labelColors := range valueMap { + if _, found := dedupedColors[labelName][labelVal]; !found { + dedupedColors[labelName][labelVal] = labelColors + } + } + } + } + + return dedupedColors +} + +// DedupAutocomplete returns a list of autocomplete hints merged from all +// Alertmanager upstreams +func DedupAutocomplete() []models.Autocomplete { + dedupedAutocomplete := []models.Autocomplete{} + uniqueAutocomplete := map[string]*models.Autocomplete{} + + upstreams := GetAlertmanagers() + + for _, am := range upstreams { + ac := am.Autocomplete() + for _, hint := range ac { + h, found := uniqueAutocomplete[hint.Value] + if found { + for _, token := range hint.Tokens { + if !stringInSlice(h.Tokens, token) { + h.Tokens = append(h.Tokens, token) + } + } + } else { + uniqueAutocomplete[hint.Value] = &models.Autocomplete{ + Value: hint.Value, + Tokens: hint.Tokens, + } + } + } + } + + for _, hint := range uniqueAutocomplete { + dedupedAutocomplete = append(dedupedAutocomplete, *hint) + } + + return dedupedAutocomplete +} diff --git a/alertmanager/mapper.go b/alertmanager/mapper.go new file mode 100644 index 000000000..e33881622 --- /dev/null +++ b/alertmanager/mapper.go @@ -0,0 +1,19 @@ +package alertmanager + +import ( + "github.com/cloudflare/unsee/mapper" + "github.com/cloudflare/unsee/mapper/v04" + "github.com/cloudflare/unsee/mapper/v05" + "github.com/cloudflare/unsee/mapper/v061" + "github.com/cloudflare/unsee/mapper/v062" +) + +// initialize all mappers +func init() { + mapper.RegisterAlertMapper(v04.AlertMapper{}) + mapper.RegisterAlertMapper(v05.AlertMapper{}) + mapper.RegisterAlertMapper(v061.AlertMapper{}) + mapper.RegisterAlertMapper(v062.AlertMapper{}) + mapper.RegisterSilenceMapper(v04.SilenceMapper{}) + mapper.RegisterSilenceMapper(v05.SilenceMapper{}) +} diff --git a/alertmanager/models.go b/alertmanager/models.go new file mode 100644 index 000000000..29cbb4cf6 --- /dev/null +++ b/alertmanager/models.go @@ -0,0 +1,281 @@ +package alertmanager + +import ( + "crypto/sha1" + "fmt" + "io" + "sort" + "sync" + "time" + + "github.com/cloudflare/unsee/config" + "github.com/cloudflare/unsee/mapper" + "github.com/cloudflare/unsee/models" + "github.com/cloudflare/unsee/transform" + "github.com/cloudflare/unsee/transport" + "github.com/cnf/structhash" + "github.com/prometheus/client_golang/prometheus" + + log "github.com/Sirupsen/logrus" +) + +// Alertmanager represents Alertmanager upstream instance +type Alertmanager struct { + URI string `json:"uri"` + Timeout time.Duration `json:"timeout"` + Name string `json:"name"` + // lock protects data access while updating + lock sync.RWMutex + // fields for storing pulled data + alertGroups []models.AlertGroup + silences map[string]models.Silence + colors models.LabelsColorMap + autocomplete []models.Autocomplete +} + +func (am *Alertmanager) detectVersion() string { + // if everything fails assume Alertmanager is at latest possible version + defaultVersion := "999.0.0" + + url, err := transport.JoinURL(am.URI, "api/v1/status") + if err != nil { + log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", am.URI, err) + return defaultVersion + } + ver := alertmanagerVersion{} + err = transport.ReadJSON(url, am.Timeout, &ver) + if err != nil { + log.Errorf("[%s] %s request failed: %s", am.Name, url, err.Error()) + return defaultVersion + } + + if ver.Status != "success" { + log.Errorf("[%s] Request to %s returned status %s", am.Name, url, ver.Status) + return defaultVersion + } + + if ver.Data.VersionInfo.Version == "" { + log.Errorf("[%s] No version information in Alertmanager API at %s", am.Name, url) + return defaultVersion + } + + log.Infof("[%s] Remote Alertmanager version: %s", am.Name, ver.Data.VersionInfo.Version) + return ver.Data.VersionInfo.Version +} + +func (am *Alertmanager) pullSilences(version string) error { + mapper, err := mapper.GetSilenceMapper(version) + if err != nil { + return err + } + + start := time.Now() + silences, err := mapper.GetSilences(am.URI, am.Timeout) + if err != nil { + return err + } + log.Infof("[%s] Got %d silences(s) in %s", am.Name, len(silences), time.Since(start)) + + log.Infof("[%s] Detecting JIRA links in silences (%d)", am.Name, len(silences)) + silenceMap := map[string]models.Silence{} + for _, silence := range silences { + silence.JiraID, silence.JiraURL = transform.DetectJIRAs(&silence) + silenceMap[silence.ID] = silence + } + + am.lock.Lock() + am.silences = silenceMap + am.lock.Unlock() + + return nil +} + +func (am *Alertmanager) pullAlerts(version string) error { + mapper, err := mapper.GetAlertMapper(version) + if err != nil { + return err + } + + start := time.Now() + groups, err := mapper.GetAlerts(am.URI, am.Timeout) + if err != nil { + return err + } + log.Infof("[%s] Got %d alert group(s) in %s", am.Name, len(groups), time.Since(start)) + + log.Infof("[%s] Deduplicating alert groups (%d)", am.Name, len(groups)) + uniqueGroups := map[string]models.AlertGroup{} + uniqueAlerts := map[string]map[string]models.Alert{} + for _, ag := range groups { + agIDHasher := sha1.New() + io.WriteString(agIDHasher, ag.Receiver) + io.WriteString(agIDHasher, fmt.Sprintf("%x", structhash.Sha1(ag.Labels, 1))) + agID := fmt.Sprintf("%x", agIDHasher.Sum(nil)) + if _, found := uniqueGroups[agID]; !found { + uniqueGroups[agID] = models.AlertGroup{ + Receiver: ag.Receiver, + Labels: ag.Labels, + ID: agID, + } + } + for _, alert := range ag.Alerts { + // generate alert id from labels + aID := fmt.Sprintf("%x", structhash.Sha1(alert.Labels, 1)) + alert.ID = aID + // generate alert fingerprint from a raw, unaltered alert object + fp := fmt.Sprintf("%x", structhash.Sha1(alert, 1)) + if _, found := uniqueAlerts[agID]; !found { + uniqueAlerts[agID] = map[string]models.Alert{} + } + if _, found := uniqueAlerts[agID][fp]; !found { + alert.Fingerprint = fp + uniqueAlerts[agID][fp] = alert + } + } + + } + + dedupedGroups := []models.AlertGroup{} + colors := models.LabelsColorMap{} + autocompleteMap := map[string]models.Autocomplete{} + log.Infof("[%s] Processing unique alert groups (%d)", am.Name, len(uniqueGroups)) + for _, ag := range uniqueGroups { + alerts := models.AlertList{} + for _, alert := range uniqueAlerts[ag.ID] { + + silences := map[string]models.Silence{} + for _, silenceID := range alert.SilencedBy { + silence, err := am.SilenceByID(silenceID) + if err == nil { + silences[silenceID] = silence + } + } + alert.Alertmanager = []models.AlertmanagerUpstream{ + models.AlertmanagerUpstream{ + Name: am.Name, + URI: am.URI, + Silences: silences, + }, + } + + alert.Annotations, alert.Links = transform.DetectLinks(alert.Annotations) + alert.Labels = transform.StripLables(config.Config.StripLabels, alert.Labels) + + transform.ColorLabel(colors, "@receiver", alert.Receiver) + for k, v := range alert.Labels { + transform.ColorLabel(colors, k, v) + } + alerts = append(alerts, alert) + + // update internal metrics + metricAlerts.With(prometheus.Labels{ + "alertmanager": am.Name, + "state": alert.State, + }).Inc() + } + + for _, hint := range transform.BuildAutocomplete(alerts) { + autocompleteMap[hint.Value] = hint + } + + sort.Sort(&alerts) + ag.Alerts = alerts + + // Hash is a checksum of all alerts, used to tell when any alert in the group changed + ag.Hash = ag.ContentFingerprint() + + dedupedGroups = append(dedupedGroups, ag) + } + + log.Infof("[%s] Merging autocomplete data (%d)", am.Name, len(autocompleteMap)) + autocomplete := []models.Autocomplete{} + for _, hint := range autocompleteMap { + autocomplete = append(autocomplete, hint) + } + + am.lock.Lock() + am.alertGroups = dedupedGroups + am.colors = colors + am.autocomplete = autocomplete + am.lock.Unlock() + + metricAlertGroups.With(prometheus.Labels{ + "alertmanager": am.Name, + }).Set(float64(len(dedupedGroups))) + + return nil +} + +// Pull data from upstream Alertmanager instance +func (am *Alertmanager) Pull() error { + version := am.detectVersion() + + err := am.pullSilences(version) + if err != nil { + metricAlertmanagerErrors.With(prometheus.Labels{ + "alertmanager": am.Name, + "endpoint": "silences", + }).Inc() + return err + } + + err = am.pullAlerts(version) + if err != nil { + metricAlertmanagerErrors.With(prometheus.Labels{ + "alertmanager": am.Name, + "endpoint": "alerts", + }).Inc() + return err + } + + return nil +} + +// Alerts returns a copy of all alert groups +func (am *Alertmanager) Alerts() []models.AlertGroup { + am.lock.RLock() + defer am.lock.RUnlock() + + alerts := make([]models.AlertGroup, len(am.alertGroups)) + copy(alerts, am.alertGroups) + return alerts +} + +// SilenceByID allows to query for a silence by it's ID, returns error if not found +func (am *Alertmanager) SilenceByID(id string) (models.Silence, error) { + am.lock.RLock() + defer am.lock.RUnlock() + + for k, v := range am.silences { + if k == id { + return models.Silence(v), nil + } + } + + return models.Silence{}, fmt.Errorf("Silence '%s' not found", id) +} + +// Colors returns a copy of all color maps +func (am *Alertmanager) Colors() models.LabelsColorMap { + am.lock.RLock() + defer am.lock.RUnlock() + + colors := models.LabelsColorMap{} + for k, v := range am.colors { + colors[k] = map[string]models.LabelColors{} + for nk, nv := range v { + colors[k][nk] = nv + } + } + return colors +} + +// Autocomplete returns a copy of all autocomplete data +func (am *Alertmanager) Autocomplete() []models.Autocomplete { + am.lock.RLock() + defer am.lock.RUnlock() + + autocomplete := make([]models.Autocomplete, len(am.autocomplete)) + copy(autocomplete, am.autocomplete) + return autocomplete +} diff --git a/alertmanager/slices.go b/alertmanager/slices.go new file mode 100644 index 000000000..f74dea2e0 --- /dev/null +++ b/alertmanager/slices.go @@ -0,0 +1,19 @@ +package alertmanager + +func boolInSlice(boolArray []bool, value bool) bool { + for _, s := range boolArray { + if s == value { + return true + } + } + return false +} + +func stringInSlice(stringArray []string, value string) bool { + for _, s := range stringArray { + if s == value { + return true + } + } + return false +} diff --git a/alertmanager/upstream.go b/alertmanager/upstream.go index f2955341e..178a70eee 100644 --- a/alertmanager/upstream.go +++ b/alertmanager/upstream.go @@ -1,40 +1,26 @@ package alertmanager import ( - "crypto/sha1" "fmt" - "io" - "sort" "sync" "time" - "github.com/cloudflare/unsee/config" - "github.com/cloudflare/unsee/mapper" "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/transform" - "github.com/cloudflare/unsee/transport" - "github.com/cnf/structhash" "github.com/prometheus/client_golang/prometheus" log "github.com/Sirupsen/logrus" ) -// Alertmanager represents Alertmanager upstream instance -type Alertmanager struct { - URI string `json:"uri"` - Timeout time.Duration `json:"timeout"` - Name string `json:"name"` - // lock protects data access while updating - lock sync.RWMutex - // fields for storing pulled data - alertGroups []models.AlertGroup - silences map[string]models.Silence - colors models.LabelsColorMap - autocomplete []models.Autocomplete -} +var ( + upstreams = map[string]*Alertmanager{} +) // NewAlertmanager creates a new Alertmanager instance -func NewAlertmanager(name, uri string, timeout time.Duration) Alertmanager { +func NewAlertmanager(name, uri string, timeout time.Duration) error { + if _, found := upstreams[name]; found { + return fmt.Errorf("Alertmanager upstream '%s' already exist", name) + } + // initialize metrics metricAlertmanagerErrors.With(prometheus.Labels{ "alertmanager": name, @@ -45,7 +31,7 @@ func NewAlertmanager(name, uri string, timeout time.Duration) Alertmanager { "endpoint": "silences", }).Set(0) - return Alertmanager{ + upstreams[name] = &Alertmanager{ URI: uri, Timeout: timeout, Name: name, @@ -55,217 +41,27 @@ func NewAlertmanager(name, uri string, timeout time.Duration) Alertmanager { colors: models.LabelsColorMap{}, autocomplete: []models.Autocomplete{}, } -} -func (am *Alertmanager) detectVersion() string { - // if everything fails assume Alertmanager is at latest possible version - defaultVersion := "999.0.0" - - url, err := transport.JoinURL(am.URI, "api/v1/status") - if err != nil { - log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", am.URI, err) - return defaultVersion - } - ver := alertmanagerVersion{} - err = transport.ReadJSON(url, am.Timeout, &ver) - if err != nil { - log.Errorf("[%s] %s request failed: %s", am.Name, url, err.Error()) - return defaultVersion - } - - if ver.Status != "success" { - log.Errorf("[%s] Request to %s returned status %s", am.Name, url, ver.Status) - return defaultVersion - } - - if ver.Data.VersionInfo.Version == "" { - log.Errorf("[%s] No version information in Alertmanager API at %s", am.Name, url) - return defaultVersion - } - - log.Infof("[%s] Remote Alertmanager version: %s", am.Name, ver.Data.VersionInfo.Version) - return ver.Data.VersionInfo.Version -} - -func (am *Alertmanager) pullSilences(version string) error { - mapper, err := mapper.GetSilenceMapper(version) - if err != nil { - return err - } - - start := time.Now() - silences, err := mapper.GetSilences() - if err != nil { - return err - } - log.Infof("[%s] Got %d silences(s) in %s", am.Name, len(silences), time.Since(start)) - - log.Infof("[%s] Detecting JIRA links in silences (%d)", am.Name, len(silences)) - silenceMap := map[string]models.Silence{} - for _, silence := range silences { - silence.JiraID, silence.JiraURL = transform.DetectJIRAs(&silence) - silenceMap[silence.ID] = silence - } - - am.lock.Lock() - am.silences = silenceMap - am.lock.Unlock() + log.Infof("[%s] Configured Alertmanager source at %s", name, uri) return nil } -func (am *Alertmanager) pullAlerts(version string) error { - mapper, err := mapper.GetAlertMapper(version) - if err != nil { - return err +// GetAlertmanagers returns a list of all defined Alertmanager instances +func GetAlertmanagers() []*Alertmanager { + ams := []*Alertmanager{} + for _, am := range upstreams { + ams = append(ams, am) } + return ams +} - start := time.Now() - groups, err := mapper.GetAlerts() - if err != nil { - return err +// GetAlertmanagerByName returns an instance of Alertmanager by name or nil +// if not found +func GetAlertmanagerByName(name string) *Alertmanager { + am, found := upstreams[name] + if found { + return am } - log.Infof("[%s] Got %d alert group(s) in %s", am.Name, len(groups), time.Since(start)) - - log.Infof("[%s] Deduplicating alert groups (%d)", am.Name, len(groups)) - uniqueGroups := map[string]models.AlertGroup{} - uniqueAlerts := map[string]map[string]models.Alert{} - for _, ag := range groups { - agIDHasher := sha1.New() - io.WriteString(agIDHasher, ag.Receiver) - io.WriteString(agIDHasher, fmt.Sprintf("%x", structhash.Sha1(ag.Labels, 1))) - agID := fmt.Sprintf("%x", agIDHasher.Sum(nil)) - if _, found := uniqueGroups[agID]; !found { - uniqueGroups[agID] = models.AlertGroup{ - Receiver: ag.Receiver, - Labels: ag.Labels, - ID: agID, - } - } - for _, alert := range ag.Alerts { - // generate alert fingerprint from a raw, unaltered alert object - aID := fmt.Sprintf("%x", structhash.Sha1(alert, 1)) - if _, found := uniqueAlerts[agID]; !found { - uniqueAlerts[agID] = map[string]models.Alert{} - } - if _, found := uniqueAlerts[agID][aID]; !found { - alert.Fingerprint = aID - uniqueAlerts[agID][aID] = alert - } - } - - } - - dedupedGroups := []models.AlertGroup{} - colors := models.LabelsColorMap{} - autocompleteMap := map[string]models.Autocomplete{} - log.Infof("[%s] Processing unique alert groups (%d)", am.Name, len(uniqueGroups)) - for _, ag := range uniqueGroups { - // used to generate group content hash - agHasher := sha1.New() - - alerts := models.AlertList{} - for _, alert := range uniqueAlerts[ag.ID] { - - alert.Annotations, alert.Links = transform.DetectLinks(alert.Annotations) - alert.Labels = transform.StripLables(config.Config.StripLabels, alert.Labels) - - io.WriteString(agHasher, alert.Fingerprint) // alert group hasher - - transform.ColorLabel(colors, "@receiver", alert.Receiver) - for k, v := range alert.Labels { - transform.ColorLabel(colors, k, v) - } - alerts = append(alerts, alert) - - // update internal metrics - metricAlerts.With(prometheus.Labels{ - "alertmanager": am.Name, - "state": alert.State, - }).Inc() - } - - for _, hint := range transform.BuildAutocomplete(alerts) { - autocompleteMap[hint.Value] = hint - } - - sort.Sort(&alerts) - ag.Alerts = alerts - - // Hash is a checksum of all alerts, used to tell when any alert in the group changed - ag.Hash = fmt.Sprintf("%x", agHasher.Sum(nil)) - - dedupedGroups = append(dedupedGroups, ag) - } - - log.Infof("[%s] Merging autocomplete data (%d)", am.Name, len(autocompleteMap)) - autocomplete := []models.Autocomplete{} - for _, hint := range autocompleteMap { - autocomplete = append(autocomplete, hint) - } - - am.lock.Lock() - am.alertGroups = dedupedGroups - am.colors = colors - am.autocomplete = autocomplete - am.lock.Unlock() - - metricAlertGroups.With(prometheus.Labels{ - "alertmanager": am.Name, - }).Set(float64(len(dedupedGroups))) - return nil } - -// Pull data from upstream Alertmanager instance -func (am *Alertmanager) Pull() error { - version := am.detectVersion() - - err := am.pullSilences(version) - if err != nil { - metricAlertmanagerErrors.With(prometheus.Labels{ - "alertmanager": am.Name, - "endpoint": "silences", - }).Inc() - return err - } - - err = am.pullAlerts(version) - if err != nil { - metricAlertmanagerErrors.With(prometheus.Labels{ - "alertmanager": am.Name, - "endpoint": "alerts", - }).Inc() - return err - } - - return nil -} - -// Alerts returns a copy of all stored alert groups -func (am *Alertmanager) Alerts() []models.AlertGroup { - am.lock.RLock() - defer am.lock.RUnlock() - - alerts := make([]models.AlertGroup, len(am.alertGroups)) - copy(alerts, am.alertGroups) - return alerts -} - -// SilencesByID returns a map copy of id->silences for given list of silence IDs -func (am *Alertmanager) SilencesByID(ids []string) map[string]models.Silence { - am.lock.RLock() - defer am.lock.RUnlock() - - silences := map[string]models.Silence{} - for k, v := range am.silences { - for _, s := range ids { - if k == s { - silences[k] = v - break - } - } - } - - return silences -} diff --git a/alertmanager/version.go b/alertmanager/version.go index 558d58f49..6daac1f65 100644 --- a/alertmanager/version.go +++ b/alertmanager/version.go @@ -1,6 +1,8 @@ package alertmanager import ( + "time" + "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/transport" @@ -19,17 +21,17 @@ type alertmanagerVersion struct { } // GetVersion returns version information of the remote Alertmanager endpoint -func GetVersion() string { +func GetVersion(uri string, timeout time.Duration) string { // if everything fails assume Alertmanager is at latest possible version defaultVersion := "999.0.0" - url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/status") + url, err := transport.JoinURL(uri, "api/v1/status") if err != nil { log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", config.Config.AlertmanagerURI, err.Error()) return defaultVersion } ver := alertmanagerVersion{} - err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &ver) + err = transport.ReadJSON(url, timeout, &ver) if err != nil { log.Errorf("%s request failed: %s", url, err.Error()) return defaultVersion diff --git a/assets/static/alerts.js b/assets/static/alerts.js index f1b31fd68..52a1abf05 100644 --- a/assets/static/alerts.js +++ b/assets/static/alerts.js @@ -6,8 +6,7 @@ /* exported Alerts */ var Alerts = (function() { - var silences = {}, - labelCache = new LRUMap(1000); + var labelCache = new LRUMap(1000); function AlertGroup(groupData) { $.extend(this, groupData); @@ -16,7 +15,6 @@ var Alerts = (function() { AlertGroup.prototype.Render = function() { return Templates.Render("alertGroup", { group: this, - silences: silences, alertLimit: 5 }); }; @@ -105,9 +103,6 @@ var Alerts = (function() { var alertCount = 0; var groups = {}; - // update global silences dict as it's needed for rendering - silences = apiResponse.silences; - var summaryData = {}; $.each(apiResponse.counters, function(labelKey, counters){ $.each(counters, function(labelVal, hits){ diff --git a/assets/templates/alertgroup.html b/assets/templates/alertgroup.html index f60c16a53..94b85bd57 100644 --- a/assets/templates/alertgroup.html +++ b/assets/templates/alertgroup.html @@ -110,10 +110,9 @@ Silenced by: - <% _.each(alert.silencedBy, function(silenceID) { %> -
- <% var silence = silences[silenceID] %> - <% if (silence) { %> + <% _.each(alert.alertmanager, function(am) { %> + <% _.each(am.silences, function(silence) { %> +
<% if (silence.jiraURL) { %> @@ -124,6 +123,9 @@ <%- silence.comment %> <% } %>
+
+ <%- am.name %> +
- <% } %> -
+
+ <% }) %> <% }) %> @@ -261,7 +263,7 @@ <%= Templates.Render('alertGroupLabels', {alert: alert, group: group}) %> <%= Templates.Render('alertGroupElements', {alert: alert}) %> <% if (alert.silencedBy.length) { %> - <%= Templates.Render('alertGroupSilence', {alert: alert, silences: silences}) %> + <%= Templates.Render('alertGroupSilence', {alert: alert}) %> <% } %> diff --git a/bindata_assetfs.go b/bindata_assetfs.go index 1c873ea7b..609a5626b 100644 --- a/bindata_assetfs.go +++ b/bindata_assetfs.go @@ -138,7 +138,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _templatesAlertgroupHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\x5f\x6f\xdb\x38\x12\x7f\xef\xa7\x98\xd5\x6d\x56\x09\x62\xd9\xdd\xb7\x83\x61\xf9\x36\xbb\x57\x14\x87\x6b\xb1\x40\xdb\x7b\x2a\x8a\x82\x96\xc6\x36\x1b\x59\x52\x49\x2a\x5b\x23\xc8\x7d\xf6\x03\xff\x88\x22\x29\xca\x71\xb6\xd9\x87\xeb\x43\x63\x91\xc3\xe1\x70\xfe\xf1\xc7\x21\x57\xbc\x60\xb4\x15\x20\x8e\x2d\xe6\x09\x69\xdb\x8a\x16\x44\xd0\xa6\x5e\x7c\xe1\x4d\x9d\x00\x2d\xf3\x84\x54\xc8\x44\xb6\x63\x4d\xd7\x66\x82\x8a\x0a\x93\xf5\x0b\x80\xd5\x05\xd0\x2d\x5c\xfe\xbe\xf9\x82\x85\x98\xdf\xe2\x91\x5f\x2a\x92\x79\x45\x36\x58\xf1\xab\x79\x85\xf5\x4e\xec\x61\x0d\x2f\xaf\xe0\x1e\x2e\xe4\x18\x35\xea\x8e\x30\xd8\xd2\x4a\x20\xe3\x90\xc3\xc7\xf4\x17\x86\x05\xd2\x3b\x64\x79\x0a\xd7\xa0\x79\xf4\x4d\x9f\x9c\x71\x9f\xe7\x48\x8a\xbd\x37\xc9\x0c\xb6\x5d\x5d\x48\x71\x2f\x55\xc3\xe7\x3b\x52\xcd\x40\xff\xbc\xc5\xa3\x9c\xd7\xcc\x34\x6f\x3b\xbe\xbf\xb4\x3d\x70\x0d\xa9\x9a\xce\x0e\xbb\x82\x87\xab\x40\x48\x35\xd3\x1b\x5a\xdf\x42\x0e\xe9\x3f\xbe\x2a\xfa\x9e\xdd\x97\x86\xd6\x97\xe9\x2c\x1d\xc6\xf0\x96\xd4\x50\x54\x84\xf3\x3c\x69\xbb\xaa\xca\x2a\xdc\x0a\x70\x75\x57\xd1\xfa\x36\xd1\xd4\x00\x2b\x02\x7b\x86\xdb\x3c\x59\x5d\xe4\xce\x4c\x17\xeb\x04\x94\x8e\xf3\x44\x7d\x8b\x06\xc4\x9e\x72\xcd\x47\xd3\x25\x33\x28\x89\x20\x99\x68\x76\x3b\x49\x27\x9a\xa6\x12\xb4\x4d\x74\x6b\x5b\x91\x02\x0f\x58\x0b\xd9\xd1\xda\xe9\x00\x56\xb4\x97\x6e\x4b\x60\x4b\xb4\x34\x0b\x2b\xce\x82\x98\x75\x2c\xe4\x42\xac\x1e\xa4\x89\xb5\xc6\x95\x04\x7c\xb0\xea\xcf\xae\x55\xdd\xc5\x6f\x48\xb9\x43\x50\x2a\x60\x74\xb7\x17\xc3\x92\x2f\x32\x88\xf1\xba\x88\xce\xfc\x60\xdb\x4b\x7a\x67\x15\x4b\x6a\xac\x1c\x27\x74\xa4\x7c\xa2\x23\xfa\x4e\x75\xa3\x05\x7a\xdf\x30\xf1\x96\xb4\xbf\x1e\xff\x8d\x47\x9f\x49\xe8\x69\x01\x27\xeb\x33\x44\x08\xe5\xd6\x86\xe1\x6b\x14\x6f\x24\xf9\x8d\x6c\xd6\x23\xa5\x84\xc6\x47\xe7\x77\xa4\xea\xf0\x2a\x64\x94\xc3\x07\x3c\xb4\x15\x11\xc8\xe7\xef\xb0\x2e\x91\x5d\xa6\x9b\x4e\x88\xa6\x56\xbc\xd2\x19\xdc\x63\x85\x87\x25\xa4\x25\xbd\x4b\x67\x20\x3f\x7e\x93\xfa\x59\x42\xaa\xf8\x6a\xee\x59\x45\xb9\x48\x67\x5a\xa4\xa5\xfe\x63\x26\x5e\xea\x3f\x0f\x57\x81\x36\xdc\x06\x65\x02\xac\x38\x06\x3a\x73\xac\x31\x4c\xa3\xf9\x25\xeb\xd5\xa2\xa4\x77\x1e\x07\x6b\x5c\xd3\xd1\x37\xae\x16\x3a\xf3\xac\x5f\xbc\x78\x6a\x0e\x22\x75\xdd\x08\x45\xc0\xfb\x4c\x74\xc2\x8c\x6a\xe4\xdc\x19\xe3\xda\x72\x68\x76\xbd\xd9\x59\xe2\x1f\x58\x55\x20\xff\xcb\xf8\x01\x06\xea\x4c\x36\x0d\x1e\x18\x84\xd6\xd7\x0e\xb9\xa2\x2a\x28\x2b\x2a\x04\x81\xdf\x44\x76\xe8\x04\x96\x36\xb8\xa5\x95\x07\x76\xd2\x27\x54\xe8\x9f\x1f\xd8\x8b\xc0\xfd\x1d\x66\xc6\xab\x02\x57\xcf\x20\x24\x79\xdc\xd6\x6e\x58\x3b\x8b\x70\xdd\xf5\x23\x1c\x28\xe7\xb4\xde\x39\xec\x41\xb3\xff\x34\x30\x72\x42\xfb\x84\x5f\x18\xef\x1b\xec\xa9\x6d\x27\x33\x95\x9b\xea\x3b\x56\xcd\x94\x4e\x5d\x9b\x11\xcf\x29\x21\x74\xcd\xac\xc4\x2d\xe9\x2a\x91\xf4\x32\x0d\xc9\xb7\x63\x95\xd4\x7d\xdf\x21\x08\xdb\xa1\xc8\x93\xcf\x9b\x8a\xd4\xb7\x43\xf3\x60\xb7\x60\x40\xd4\x66\x5e\xe7\x44\x4e\x0e\xdd\x06\xbf\x09\x64\x35\xa9\xc2\xd4\x7c\x91\xa9\xd5\x0e\x2a\x23\x9e\xc2\xbe\x23\x92\x74\x72\x3b\x3b\x88\x1e\xcf\x85\xde\x8e\xa1\xc9\x3f\xda\xac\xf7\x09\xf2\x1c\xba\xba\xc4\x2d\xad\xb1\xf4\x1c\xf4\x59\xb2\xe7\x53\x32\xa7\xf4\xc7\x33\x73\xe3\xe0\xae\xcf\xa1\x72\x39\x3f\xd6\xc2\x2a\x5d\x2e\xbb\xa8\xf8\x67\x5a\x97\x72\x60\xc3\x24\xd8\xa0\x75\x41\x4b\xac\x45\x66\x5b\xb3\x92\xd4\x3b\x64\xa9\x15\x44\xc5\xbc\xb2\x0a\x17\x44\x20\xe4\x79\x0e\x09\xef\xda\x96\x21\xe7\x58\x26\x52\xbd\xe7\xf0\xe5\x5d\x51\x20\xe7\xa9\x5d\xa3\x4d\xe0\xde\x24\x3b\xac\x91\xc9\x01\xff\x79\xf7\xc6\xb7\xdc\x53\x03\xcf\x0d\xbd\x31\x6f\x37\xb0\x26\x63\xd1\x46\xe3\xeb\x46\x03\x24\x34\xf8\x88\x37\x1d\x2b\xd0\xa1\x3b\x19\x9b\xa7\xa3\x73\x1c\x9f\x45\x53\x62\x0c\x31\xb9\xe9\xec\x0c\x3f\x4e\x7e\x51\x16\x4b\x66\xe0\xd8\xcf\xf1\xb6\xe7\x71\xe2\xfb\x5b\x3c\x2e\x21\xd5\x73\xa5\x33\x9d\x92\x97\xee\x94\x3a\x87\x5a\x9a\x25\x48\x7c\xeb\xf4\x3f\x78\x21\xe0\xf8\x1b\xad\xb0\x2e\xb0\xfc\xf5\xd8\x23\xab\x3c\x0f\xa0\x95\x51\x83\x4e\x00\x12\xe4\x7f\xf2\xfa\x9e\x21\xd1\x18\x4e\x9a\xd0\x01\xf7\xf3\x11\xb8\x8f\xa4\x09\x1f\xe7\xb8\x9b\xdc\x84\x0f\x9b\x20\x81\xa2\x63\xbc\x61\x59\xdb\xd0\x5a\x20\x73\x1c\x49\xba\xa4\x4a\x00\xda\x50\x7e\x8f\xf2\x32\x2d\xa9\x76\x7b\x23\xb5\x7b\x86\x88\x8c\x50\xaa\xa8\xc9\x01\xdd\x58\x31\x43\x6d\x5f\x7c\x68\xef\xf1\x87\xa6\x24\x55\xac\xdf\xc4\xd5\xdf\x8c\x2d\xdf\x2a\x3a\x0f\x85\x06\xbe\xbf\x51\x10\xa8\x22\x7c\x6f\x21\xcc\x7b\x3d\xd6\x39\x9f\x3c\x05\xbf\x80\x13\x48\x13\xb8\xff\xac\x58\xea\x4f\x8a\x36\x9c\xfa\x86\xbf\x2c\xa2\xfa\x09\xc2\xa0\xea\xdb\x87\xb8\xea\x5b\xdc\xd0\xea\xdb\xdc\xe8\x3a\x33\x85\x9a\x2f\xb2\x43\xf3\x4b\xf0\x3f\x0f\x42\xfc\x81\xdc\xf5\x31\x2e\x08\x13\xfc\x46\x02\x8e\xc1\x27\xc6\x61\x92\x09\x9e\xc9\xc6\x24\x40\x99\x21\x0f\xcb\xc1\x03\x81\x1e\xca\x1e\xef\x4b\xb0\xa7\x65\x89\xb5\x32\x9e\xbf\x8d\x79\x42\x0d\xc7\x8d\x1e\x15\xe9\x96\xef\xd8\xa0\x4d\x48\xe8\xfd\x99\x1f\x48\x55\xf5\x52\x9a\x9e\xac\x68\x0e\x52\x8f\xfa\x18\x0a\x23\x5c\x6c\xe2\xa2\x84\xcd\x71\xa9\x24\x52\x4c\x62\xc0\x76\xc8\xa4\x4e\xa6\x33\x8d\xff\xfa\xe7\xc4\x89\xa4\x97\x62\x53\x35\xc5\x6d\x12\x64\x5c\xd3\x09\x79\xff\x8b\x7f\xb4\xfc\xfc\x1c\x2c\xf3\xb9\xe9\x0a\xd3\xaa\xe2\xfc\xb5\x6b\x04\x4e\xac\x3c\x09\x0e\xbe\x0e\xaf\xf9\x17\xca\x48\x88\x11\x0c\xa1\x5b\xf1\x08\xc8\x75\xdd\xc3\xdf\xed\xfd\xd1\x67\xc3\x65\xd7\x17\xfb\x59\x8c\xdc\x23\x89\xfa\x2d\xdc\xae\x24\x76\x0a\x3a\x8b\x99\x9b\xb8\x7a\x35\x32\x4f\xa6\xd1\xc1\xf9\x29\x71\xde\xef\x3c\x7b\xac\x5a\x3f\x9f\x3f\x06\x71\xe0\x91\xf8\x77\x99\x70\xdf\x36\xd1\x3c\xa0\xff\xbd\x97\x5d\x58\xfa\x8d\x67\xa5\x88\xb1\x3e\x23\xa9\xa2\x37\x8f\x97\x30\xfc\x78\xff\x7f\x55\x2a\xd6\x65\x5c\xa5\xaf\xea\x92\x3f\x8b\x3e\xed\x0c\x4f\xd7\xa6\xcd\x4f\x3d\xaf\x03\x11\xc5\x1e\x99\x7b\xfa\x36\x4d\xb1\x00\x7f\xdc\x18\x06\x48\x45\xe4\x57\xe9\xcb\xf0\xfe\xbd\xd5\xe7\x01\xc8\x21\xc9\x93\xd1\x4a\xfa\x94\x63\xa8\xe7\x94\xbf\xc3\x1d\x7e\x8b\x48\x64\x88\x63\x6c\xff\x1b\xe7\xfb\x10\x6b\xcd\x7a\x0e\x73\x03\xb9\x9c\x26\xcb\xd4\x6b\x0d\x0b\x2c\xd3\x0a\x7f\x08\xcb\x7f\xdb\xa6\x11\xc8\x82\x81\x05\x15\x38\x92\x8b\x6c\x36\x2c\xb6\x5e\x27\x55\x31\x24\x42\xee\x2f\x91\x55\x2d\xc6\xc3\x57\x8b\x70\x9e\xd5\x22\x14\x67\xb5\x18\xb6\x87\xf3\x0a\x3a\xdf\x5b\x9f\xc8\x0e\xa4\x75\x4f\xcb\xaa\xf1\x86\x31\xe7\x5c\x31\x38\xae\xea\x7c\x4b\xda\xf0\xe8\x30\x2e\x18\x5d\x58\x46\xfa\xf8\x70\xaf\x51\x9b\xfc\x7f\x06\x0a\xea\x39\x05\x07\x03\xf4\x9c\x03\xc5\x0c\xf6\x54\xf0\xbe\x49\xfe\x76\xca\x57\xce\x4f\x3b\x09\x6f\x98\xb8\x1c\x0a\x8f\x33\xd8\x8c\xeb\x26\x44\x31\x82\x15\x6c\xd4\x8f\x2b\x60\x28\x3a\x56\xc3\xcf\x71\xba\x75\x48\x97\x8d\x09\x55\xe5\x48\x12\xea\xf5\x4f\x32\x54\x74\xab\x90\xce\x63\x68\xda\x5e\x06\xcb\x7c\x11\x2b\xd1\x6f\x9a\xf2\x08\x16\xd6\x19\x64\x85\x2d\x51\xb1\x92\x8c\x51\x8d\x4f\x3a\x82\x35\xb7\xb4\x6d\xb1\x54\x68\x1d\x72\x48\xaf\x25\x9e\x36\x8d\x63\x54\x63\xda\xf3\xdc\xbb\xa3\x30\x14\x1e\xab\xeb\x1c\x92\xfe\xe8\xf2\x68\x31\x74\x72\x2c\x0f\x07\x0f\x5f\x8f\xaf\x30\x76\x79\x12\xe0\xab\xcc\x9f\xd8\x95\x29\xc8\xe9\x46\x5d\x4c\x9d\x71\xa4\x0a\x7a\x63\x99\xee\x1f\xb2\xcc\x7e\x14\xa4\x4e\x05\x74\x1c\x55\x49\x8e\xf1\xa2\x61\x08\x2a\x86\xae\x60\x8f\x0c\x81\x70\xa0\x02\xca\x06\xb9\x24\xe4\x5d\xdb\x36\x4c\xc0\x86\x21\xb9\xa5\xf5\x4e\x55\x5d\xaa\xa6\x69\x2d\xc3\x3f\xa8\xd8\x43\x6a\x9c\x64\x4b\x2a\x8e\xe9\x4c\xf1\xff\xf2\xb5\x43\x76\x84\x03\x8a\x7d\x33\x40\x86\x2c\xf3\xe4\xfe\xd1\x89\xdf\x1b\xc6\x9c\xf8\xa5\xe6\x04\x16\xbb\x72\x91\xf6\xb6\xab\x5d\xc3\xdf\xe1\xa7\x9f\x86\x88\x1b\xae\xaa\xc2\x8b\x9f\x91\xda\x1f\xa9\x5d\x4d\x6c\x57\xc6\x2c\xbf\x35\x5d\x2d\x20\x1f\x4d\x9c\x0d\x86\x98\xda\xc2\x7c\x0e\x63\x87\xed\xff\x5d\x3b\x5e\xa0\x89\x2f\xd6\x7a\xbe\xd8\x16\x16\xc7\xb0\xa7\xf9\xf0\x33\xf6\xc2\x31\x84\x70\xf3\x82\x32\x79\x04\x15\x4f\x21\x6a\xab\x9d\xeb\xeb\x48\xe7\xf3\x5c\xa6\x59\x45\x0f\x69\x3a\xb8\xba\x74\x28\x47\x37\x5b\x82\xd1\xc3\x28\x5c\x07\xc6\x4f\x2e\x2b\x4c\x5d\xd4\xa9\x79\x04\xd9\x3d\xe1\xb2\xce\x4a\x31\xe5\xc5\x3e\xec\x55\x33\x14\xd2\xe0\x91\x95\xe8\x1c\xe3\x68\x28\x36\x4f\xc4\xf4\x11\x40\xf3\x88\x03\xfe\x45\x15\x7a\x6f\xee\x93\xc7\xb2\xb0\xf6\xe7\xd6\x0f\x7a\xdc\xf2\xbd\x15\x84\x64\xfd\x22\x9e\xf9\x35\xa5\xbd\xef\x9f\xd3\x72\xb8\xf3\xdb\x13\xbe\x77\xfb\xe4\xf7\x70\x48\x70\xee\x09\xd4\xf6\x2a\xf7\x40\xbd\xcf\x1a\x33\xa7\xd1\x0b\x18\x55\xc6\x55\x61\x3e\x27\x85\xa0\x77\x38\xba\x03\x5f\x5d\x44\x99\xba\x77\x0d\xae\x49\xa3\xac\x87\x4b\x87\x33\xd9\xdb\x2b\x87\xd1\x05\x4b\x04\x49\xd8\x0a\x90\xfe\x72\xce\x4d\x63\xd0\xb1\x47\x52\xaa\xad\x09\xbf\x89\xac\x40\x55\xa3\x75\x6d\x1f\x73\x3e\x65\xb8\xd7\x72\x4d\x1f\xa8\xa8\xe4\x96\x75\xaf\x56\xb8\xd4\x86\x98\xf0\x16\x1f\x8c\xbe\x25\x2d\xe4\x70\xff\x30\x2a\x80\x5b\x28\xe2\xee\xc3\xe1\x53\x16\x8d\x21\xdc\x4b\x69\xd9\x30\x03\x3a\x06\x2f\x52\xff\x14\xd6\x1a\x76\xbc\xa1\x07\x2a\x20\x8b\x64\xb3\x01\xaa\x84\xa9\x75\x74\xb5\x7a\xee\x33\x9a\x78\x82\x56\xa0\x31\x07\xef\x59\x8d\x2e\x76\x5a\x3e\x13\xf9\x78\x7c\x43\xf8\xf9\xf4\x0d\x61\x30\xbe\xd7\xfb\x47\x29\xc2\x78\x58\xd8\x6d\x8a\xb7\x76\x22\x1f\xd1\xeb\xf5\x6a\x3c\xff\xf2\x21\x7e\x06\x0c\x58\xaa\x44\x19\xdd\xb8\x22\x19\x68\xf4\xc2\x43\x3b\x87\x0c\xa1\x0f\x5a\x83\x53\x77\x30\x51\xc0\x63\x25\xb1\x1c\x4e\x28\xc0\xa5\x39\xeb\x52\x68\x4a\x0d\xae\x0a\x06\xa6\x11\x3d\x4c\xee\x00\x4e\x16\x53\xc7\x83\x1c\xd2\x34\xba\x40\x0a\xab\xe8\x43\x21\xe3\xeb\x2e\x83\xa9\x23\x46\x3a\x12\x3f\x7e\x44\xe9\xb3\x8b\xfa\x18\x15\x65\xce\xc2\xef\xbd\xe8\xa7\xb3\xcb\xcd\xf0\x00\x45\xe6\x18\xd5\x61\x34\x1f\xdb\xd9\x1f\x63\xa7\x76\xcb\x11\xa7\x19\x4c\xe5\xae\x73\xf9\xbe\x32\xd7\xd3\xe7\xc9\x78\xea\x56\x70\xb2\x1c\x73\x5a\x00\x53\x3d\x1f\xaf\xac\x2f\x68\x2f\xed\xaf\x09\x99\x22\xb0\x35\x2c\xbb\xf8\x0d\xc1\x99\x6d\xf4\x0c\x4a\x2e\xf2\x87\x1f\xe7\x94\xbf\x3a\xb4\xe2\xa8\xdf\x99\xd9\x20\xbc\x1a\x65\xe9\x33\x0c\xf7\x96\xb4\x72\x81\x3d\x8f\x25\x0c\x65\x0b\x93\xb8\x97\xfd\x8f\xd1\x93\xac\x51\x9d\xe5\x4f\xe1\x15\x0d\x0b\x35\xf2\xca\xf4\xa3\x46\xf7\x55\xa7\x45\xaa\xa3\xc4\x22\x23\xd8\xe9\x95\x21\xfc\xe0\x3f\x4a\xd0\xa0\x3d\x32\xac\x87\xf3\xf7\x85\xc1\xc0\xe9\x0c\xb8\x38\x56\x32\xed\xa5\x36\xdf\xac\xa4\x06\xe5\x0c\xf2\x68\x62\x62\xaf\x6f\xd2\x93\x5e\xac\x95\x96\x15\xbb\x79\x61\x9a\x86\xda\xab\x62\x69\xee\xb3\x14\x89\x6a\xf0\x48\xe2\x77\xb5\xc3\x3d\x6d\xa6\x09\x8c\x5e\x62\x04\xb7\x78\x74\xee\x72\xfb\xf7\x5b\x31\xca\x3b\x52\xb9\x94\x7d\x61\x30\xa0\x9d\xb8\xb1\xf5\x6f\x6b\xb5\x97\x38\x77\xb5\x03\x6e\x1f\x1e\x08\xad\x16\x8e\x02\x5d\xbf\xf8\x5f\x00\x00\x00\xff\xff\xb8\x7f\x4e\xd8\xeb\x2b\x00\x00") +var _templatesAlertgroupHtml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x5a\x5f\x8f\xdb\x36\x12\x7f\xcf\xa7\x98\xea\xea\x6a\x17\x6b\xd9\xed\xdb\xc1\xb0\x7c\xdd\x16\x45\x1f\x2e\x41\x81\x24\xf7\x14\x04\x01\x2d\x8d\x6d\x66\x65\x49\x21\xa9\x6d\x8c\xc5\xde\x67\x3f\xf0\x8f\x28\x92\xa2\xbc\xde\x66\xfb\x72\x79\xd8\x58\xe4\x70\x38\x9c\x7f\xfc\x71\xc8\x35\x2f\x18\x6d\x05\x88\x53\x8b\x79\x42\xda\xb6\xa2\x05\x11\xb4\xa9\x97\x9f\x79\x53\x27\x40\xcb\x3c\x21\x15\x32\x91\xed\x59\xd3\xb5\x99\xa0\xa2\xc2\x64\xf3\x0a\x60\x3d\x03\xba\x83\xab\x3f\xb6\x9f\xb1\x10\x8b\x3b\x3c\xf1\x2b\x45\xb2\xa8\xc8\x16\x2b\x7e\xbd\xa8\xb0\xde\x8b\x03\x6c\xe0\xc7\x6b\x78\x80\x99\x1c\xa3\x46\xdd\x13\x06\x3b\x5a\x09\x64\x1c\x72\xf8\x90\xfe\xcc\xb0\x40\x7a\x8f\x2c\x4f\xe1\x06\x34\x8f\xbe\xe9\xa3\x33\xee\xd3\x02\x49\x71\xf0\x26\x99\xc3\xae\xab\x0b\x29\xee\x95\x6a\xf8\x74\x4f\xaa\x39\xe8\x9f\x77\x78\x92\xf3\x9a\x99\x16\x6d\xc7\x0f\x57\xb6\x07\x6e\x20\x55\xd3\xd9\x61\xd7\xf0\x78\x1d\x08\xa9\x66\x7a\x4d\xeb\x3b\xc8\x21\xfd\xd7\x17\x45\xdf\xb3\xfb\xdc\xd0\xfa\x2a\x9d\xa7\xc3\x18\xde\x92\x1a\x8a\x8a\x70\x9e\x27\x6d\x57\x55\x59\x85\x3b\x01\xae\xee\x2a\x5a\xdf\x25\x9a\x1a\x60\x4d\xe0\xc0\x70\x97\x27\xeb\x59\xee\xcc\x34\xdb\x24\xa0\x74\x9c\x27\xea\x5b\x34\x20\x0e\x94\x6b\x3e\x9a\x2e\x99\x43\x49\x04\xc9\x44\xb3\xdf\x4b\x3a\xd1\x34\x95\xa0\x6d\xa2\x5b\xdb\x8a\x14\x78\xc4\x5a\xc8\x8e\xd6\x4e\x07\xb0\xa6\xbd\x74\x3b\x02\x3b\xa2\xa5\x59\x5a\x71\x96\xc4\xac\x63\x29\x17\x62\xf5\x20\x4d\xac\x35\xae\x24\xe0\x83\x55\x7f\x72\xad\xea\x2e\x7e\x4b\xca\x3d\x82\x52\x01\xa3\xfb\x83\x18\x96\x3c\xcb\x20\xc6\x6b\x16\x9d\xf9\xd1\xb6\x97\xf4\xde\x2a\x96\xd4\x58\x39\x4e\xe8\x48\xf9\x4c\x47\xf4\x9d\xea\x56\x0b\xf4\xae\x61\xe2\x0d\x69\x7f\x39\xfd\x1b\x4f\x3e\x93\xd0\xd3\x02\x4e\xd6\x67\x88\x10\xca\xad\x0d\xc3\xdf\x51\xbc\x96\xe4\xb7\xb2\x59\x8f\x94\x12\x1a\x1f\x5d\xdc\x93\xaa\xc3\xeb\x90\x51\x0e\xef\xf1\xd8\x56\x44\x20\x5f\xbc\xc5\xba\x44\x76\x95\x6e\x3b\x21\x9a\x5a\xf1\x4a\xe7\xf0\x80\x15\x1e\x57\x90\x96\xf4\x3e\x9d\x83\xfc\xf8\x55\xea\x67\x05\xa9\xe2\xab\xb9\x67\x15\xe5\x22\x9d\x6b\x91\x56\xfa\x3f\x33\xf1\x4a\xff\xf7\x78\x1d\x68\xc3\x6d\x50\x26\xc0\x8a\x63\xa0\x33\xc7\x1a\xc3\x34\x9a\x5f\xb2\x59\x2f\x4b\x7a\xef\x71\xb0\xc6\x35\x1d\x7d\xe3\x7a\xa9\x33\xcf\xe6\xd5\xab\xe7\xe6\x20\x52\xd7\x8d\x50\x04\xbc\xcf\x44\x67\xcc\xa8\x46\x2e\x9c\x31\xae\x2d\x87\x66\xd7\x9b\x9d\x25\xfe\x89\x55\x05\xf2\x4f\xc6\x8f\x30\x50\x67\xb2\x69\xf0\xc0\x20\xb4\xbe\x74\xc8\x15\x55\x41\x59\x51\x21\x08\xfc\x2a\xb2\x63\x27\xb0\xb4\xc1\x2d\xad\x3c\xb0\x93\x3e\xa1\x42\xff\xf2\xc0\x5e\x06\xee\xef\x30\x33\x5e\x15\xb8\x7a\x06\x21\xc9\xd3\xb6\x76\xc3\xda\x59\x84\xeb\xae\x1f\xe0\x48\x39\xa7\xf5\xde\x61\x0f\x9a\xfd\xc7\x81\x91\x13\xda\x67\xfc\xc2\x78\xdf\x60\x4f\x6d\x3b\x99\xa9\xdc\x54\xdf\xb1\x6a\xae\x74\xea\xda\x8c\x78\x4e\x09\xa1\x6b\x66\x25\xee\x48\x57\x89\xa4\x97\x69\x48\xbe\x1d\xab\xa4\xee\xfb\x0e\x41\xd8\x1e\x45\x9e\x7c\xda\x56\xa4\xbe\x1b\x9a\x07\xbb\x05\x03\xa2\x36\xf3\x3a\x27\x72\x72\xe8\x36\xf8\x55\x20\xab\x49\x15\xa6\xe6\x59\xa6\x56\x3b\xa8\x8c\x78\x0a\xfb\x86\x48\xd2\xc9\xed\xe2\x20\x7a\x3a\x17\x7a\x3b\x86\x26\xff\x60\xb3\xde\x47\xc8\x73\xe8\xea\x12\x77\xb4\xc6\xd2\x73\xd0\x17\xc9\x9e\xcf\xc9\x9c\xd2\x1f\x2f\xcc\x8d\x83\xbb\xbe\x84\xca\xe5\xfc\x58\x0b\xab\x74\xb9\xec\xa2\xe2\x9f\x68\x5d\xca\x81\x0d\x93\x60\x83\xd6\x05\x2d\xb1\x16\x99\x6d\xcd\x4a\x52\xef\x91\xa5\x56\x10\x15\xf3\xca\x2a\x5c\x10\x81\x90\xe7\x39\x24\xbc\x6b\x5b\x86\x9c\x63\x99\x48\xf5\x5e\xc2\x97\x77\x45\x81\x9c\xa7\x76\x8d\x36\x81\x7b\x93\xec\xb1\x46\x26\x07\xfc\xe7\xed\x6b\xdf\x72\xcf\x0d\x3c\x37\xf4\xc6\xbc\xdd\xc0\x9a\x8c\x45\x1b\x8d\xbf\x37\x1a\x20\xa1\xc1\x47\xbc\xe9\x58\x81\x0e\xdd\xd9\xd8\x3c\x1f\x9d\xe3\xf8\x2c\x9a\x12\x63\x88\xc9\x4d\x67\x17\xf8\x71\xf2\xb3\xb2\x58\x32\x07\xc7\x7e\x8e\xb7\xbd\x8c\x13\x3f\xdc\xe1\x69\x05\xa9\x9e\x2b\x9d\xeb\x94\xbc\x72\xa7\xd4\x39\xd4\xd2\xac\x40\xe2\x5b\xa7\xff\xd1\x0b\x01\xc7\xdf\x68\x85\x75\x81\xe5\x2f\xa7\x1e\x59\xe5\x79\x00\xad\x8c\x1a\x74\x02\x90\x20\xff\xa3\xd7\xf7\x02\x89\xc6\x70\xd2\x84\x0e\xb8\x5f\x8c\xc0\x7d\x24\x4d\xf8\x38\xc7\xdd\xe4\x26\x7c\xd8\x04\x09\x14\x1d\xe3\x0d\xcb\xda\x86\xd6\x02\x99\xe3\x48\xd2\x25\x55\x02\xd0\x86\xf2\x7b\x94\x97\x69\x49\xb5\xdb\x1b\xa9\xdd\x33\x44\x64\x84\x52\x45\x4d\x8e\xe8\xc6\x8a\x19\x6a\xfb\xe2\x43\x7b\x8f\x3f\x36\x25\xa9\x62\xfd\x26\xae\xfe\x61\x6c\xf9\x46\xd1\x79\x28\x34\xf0\xfd\xad\x82\x40\x15\xe1\x07\x0b\x61\xde\xe9\xb1\xce\xf9\xe4\x39\xf8\x05\x9c\x40\x9a\xc0\xfd\x17\xc5\x52\x7f\x52\xb4\xe1\xd4\x37\xfc\x6d\x11\xd5\x4f\x10\x06\x55\xdf\x3e\xc4\x55\xdf\xe2\x86\x56\xdf\xe6\x46\xd7\x85\x29\xd4\x7c\x91\x3d\x9a\x5f\x82\xff\x75\x10\xe2\x0f\xe4\xae\x8f\x71\x41\x98\xe0\xb7\x12\x70\x0c\x3e\x31\x0e\x93\x4c\xf0\x4c\x36\x26\x01\xca\x0c\x79\x58\x0e\x1e\x08\xf4\x50\xf6\x78\x5f\x82\x03\x2d\x4b\xac\x95\xf1\xfc\x6d\xcc\x13\x6a\x38\x6e\xf4\xa8\x48\xb7\x7c\xc3\x06\x6d\x42\x42\xef\xcf\xfc\x48\xaa\xaa\x97\xd2\xf4\x64\x45\x73\x94\x7a\xd4\xc7\x50\x18\xe1\x62\x13\x17\x25\x6c\x4f\x2b\x25\x91\x62\x12\x03\xb6\xea\xef\x91\xd4\x64\x2f\xbd\x66\x38\x94\x1c\x7d\x44\xd5\x8f\x39\xf6\xa9\xd7\x05\xc3\xa6\xc9\x4f\xbf\x8e\x6a\x7b\xa1\xb7\x55\x53\xdc\xb9\xa6\x52\x0d\x5f\xba\x46\xe0\xc4\xfa\x92\xe0\x78\x2b\x77\x00\x43\xb2\xf8\x4c\x19\x09\x91\x80\x21\x74\xeb\x1a\x01\xb9\xae\x6e\xf8\x7b\xba\x3f\xfa\x62\x50\xec\x7a\x5c\x3f\x8b\x91\x7b\x24\x51\xbf\x51\xdb\x95\xc4\xce\x3a\x17\x31\x73\xd3\x53\xaf\x46\xe6\xc9\x34\x3a\x1e\x8f\xa3\xb9\x65\xf4\x48\xd8\x29\x19\xcf\x4d\x8e\x0b\x93\xd3\x5d\x8e\xee\x91\xfa\xc2\x29\xa6\x13\x46\xbf\x85\x1d\xb0\x6a\xfd\x8d\xe1\x29\xac\x04\x4f\x24\x12\x97\x09\xf7\xcd\x1f\x4d\x28\xfa\xdf\x3b\xd9\x85\xa5\xdf\x78\x51\xae\x19\x9b\x2c\x92\x73\x7a\x15\x7a\x99\xe7\xff\x43\xa9\x58\x97\x71\x95\xfe\x56\x97\xfc\x45\xf4\x69\x67\x78\xbe\x36\x6d\xd2\xea\x79\x1d\x89\x28\x0e\xc8\xdc\xcc\x65\x9a\x62\x39\xe4\x69\x63\x18\x44\x16\x91\x5f\x81\x06\xc3\xfb\x8f\x56\x1f\x2c\x20\x87\x24\x4f\x46\x2b\xe9\xb3\x9a\xa1\x5e\x50\xfe\x16\xf7\xf8\x35\x22\x91\x21\x8e\xb1\xfd\x6f\x9c\xef\x63\xac\x35\xeb\x39\xf4\x71\xee\x34\x59\xa6\x5e\x6b\x58\xa9\x99\x56\xf8\x63\x58\x47\xdc\x35\x8d\x40\x16\x0c\x2c\xa8\xc0\x91\x5c\x64\xbb\x65\xb1\xf5\x3a\xd9\x90\x21\x11\x12\xf2\x47\x56\xb5\x1c\x0f\x5f\x2f\xc3\x79\xd6\xcb\x50\x9c\xf5\x72\xd8\x81\x06\x28\xe8\x9e\x3e\x9d\x7a\xd0\xcb\x54\x3a\xb2\x23\x69\xdd\x73\xb7\x6a\xbc\x65\xcc\x39\xa1\x0c\x9e\xab\x3a\xdf\x90\x36\x3c\x84\x8c\x4b\x4f\x33\xcb\x48\x1f\x44\x1e\x34\xfe\x93\x7f\xe7\xa0\x40\xa3\x53\xba\x30\x90\xd1\x39\x9a\xcc\xe1\x40\x05\xef\x9b\xe4\xef\xd1\xc2\xfd\x49\x78\xc3\xc4\xd5\x80\x16\xe6\xb0\x1d\x57\x60\x88\x62\x04\x6b\xd8\xaa\x1f\xd7\xc0\x50\x74\xac\x86\x9f\xe2\x74\x9b\x90\x2e\x1b\x13\xaa\x1a\x94\x24\xd4\xeb\x9f\x64\xa8\xe8\xd6\x21\x9d\xc7\xd0\xb4\xfd\x18\x2c\xf3\x55\xac\xd8\xbf\x6d\xca\x13\x58\x80\x68\x30\x1a\xb6\x44\x05\x4b\x32\xae\xd8\xfa\xa4\x49\x70\x24\xe5\x77\xb4\x6d\xb1\x54\xb8\x1f\x72\x48\x6f\x24\x32\x37\x8d\xde\x11\x51\xa1\x1d\xd3\x9e\xe7\xde\x6d\x87\xa1\xf0\x58\xdd\xe4\x90\xf4\x87\xa0\x27\xcb\xaa\x93\x63\x79\x38\x38\x0a\xe9\x26\x56\x18\xbb\x86\x09\x30\x5c\xe6\x4f\xec\xca\x14\x24\x75\xa3\x2e\xa6\x4e\x4b\x52\x05\xbd\xb1\x4c\xf7\x77\x59\x66\x3f\x0a\x52\xa7\x02\x3a\x8e\xaa\xb8\xc7\x78\xd1\x30\x04\x15\x43\xd7\x70\x40\x86\x40\x38\x50\x01\x65\x83\x5c\x12\xf2\xae\x6d\x1b\x26\x60\xcb\x90\xdc\xd1\x7a\xaf\xea\x37\x55\xd3\xb4\x96\xe1\x9f\x54\x1c\x20\x35\x4e\xb2\x23\x15\xc7\x74\xae\xf8\x7f\xfe\xd2\x21\x3b\xc1\x11\xc5\xa1\x19\x30\x43\x96\x79\x72\x7f\xef\xc4\xef\x2d\x73\x81\x35\x35\x67\xb9\xd8\xe5\x8d\xb4\xb7\x5d\xed\x06\xfe\x09\x3f\xfc\x30\x44\xdc\x70\xe9\x15\x5e\x21\x8d\xd4\xfe\x44\x15\x6c\x62\xbf\x32\x66\xf9\xb5\xe9\x6a\x01\xf9\x68\xe2\x6c\x30\xc4\xd4\x1e\xe6\x73\x18\x3b\x6c\xff\xef\xc6\xf1\x02\x4d\x3c\xdb\xe8\xf9\x62\x7b\x58\x1c\x27\x9f\xe7\xc3\x2f\xd8\x0c\xc7\x18\xc2\xcd\x0b\xca\xe4\x11\xe4\x3d\x85\xda\xad\x76\x6e\x6e\x22\x9d\x2f\x73\x2d\x67\x15\x3d\xa4\xe9\xe0\x12\xd4\xa1\x1c\xdd\x91\x09\x46\x8f\xa3\x70\x1d\x18\x3f\xbb\x40\x31\x75\xe5\xa7\xe6\x11\x64\xff\x8c\x6b\x3f\x2b\xc5\x94\x17\xfb\xb8\x57\xcd\x50\x48\x83\x47\x56\xa2\x73\x8c\xa3\xa1\xd8\x3c\x11\xd3\x47\x10\xcd\x13\x0e\xf8\x37\xd5\xfa\xbd\xb9\xcf\x1e\xfd\xc2\x2a\xa2\x5b\x89\xe8\xaf\xb4\xbe\xb5\x16\x91\x6c\x5e\xc5\x33\xbf\xa6\xb4\x2f\x07\x16\xb4\x1c\x6e\x0f\x0f\x84\x1f\xdc\x3e\xf9\x3d\x9c\x12\x9c\x1b\x07\xb5\xbd\xca\x3d\x50\xef\xb3\xc6\xcc\x69\xf4\x2a\x47\x15\x84\x55\x98\x2f\x48\x21\xe8\x3d\x8e\x6e\xd3\xd7\xb3\x28\x53\xf7\xd6\xc2\x35\x69\x94\xf5\x70\x7d\x71\x21\x7b\x7b\x79\x31\xba\xaa\x89\x20\x09\x5b\x4b\xd2\x5f\xce\xc1\x69\x0c\x3a\x0e\x48\x4a\xb5\x35\xe1\x57\x91\x15\xa8\xaa\xbd\xae\xed\x63\xce\xa7\x0c\xf7\xbb\x5c\xd3\x7b\x2a\x2a\xb9\x65\x3d\xa8\x15\xae\xb4\x21\x26\xbc\xc5\x07\xa3\x6f\x48\x0b\x39\x3c\x3c\x8e\x4a\xe9\x16\x8a\xb8\xfb\x70\xf8\x28\x46\x63\x08\xb7\x92\x24\x1b\xe6\x40\xc7\xe0\x45\xea\x9f\xc2\x46\xc3\x8e\xd7\xf4\x48\x05\x64\x91\x6c\x36\x40\x95\x30\xb5\x8e\x2e\x69\x2f\x7d\x90\x13\x4f\xd0\x0a\x34\xe6\xe0\x3d\xd0\xd1\x65\x53\xcb\x67\x22\x1f\x8f\xef\x1a\x3f\x9d\xbf\x6b\x0c\xc6\xf7\x7a\xff\x20\x45\x18\x0f\x0b\xbb\x4d\x19\xd8\x4e\xe4\x23\x7a\xbd\x5e\x8d\xe7\x7f\x7c\x8c\x1f\x02\x03\x96\x2a\x51\x46\x37\xae\x48\x06\x1a\xbd\x15\xd1\xce\x21\x43\xe8\xbd\xd6\xe0\xd4\x6d\x4e\x14\xf0\x58\x49\x2c\x87\x33\x0a\x70\x69\x2e\xba\x5e\x9a\x52\x83\xab\x82\x81\x69\x44\x0f\x93\x3b\x80\x93\xc5\xd4\xf1\x20\x87\x34\x8d\x2e\x90\xc2\x3a\xfa\xe4\xc8\xf8\xba\xcb\x60\xea\x88\x91\x8e\xc4\x8f\x1f\x51\xfa\xec\xa2\x3e\x46\x55\x99\x8b\xf0\x7b\x2f\xfa\xf9\xec\x72\x3b\x3c\x65\x91\x39\x46\x75\x18\xcd\xc7\x76\xf6\xa7\xd8\xa9\xdd\x72\xc4\x69\x0e\x53\xb9\xeb\x52\xbe\xbf\x99\x8b\xee\xcb\x64\x3c\x77\xbf\x38\x59\x8f\x39\x2f\x80\xa9\xc3\x5f\x38\x7f\x04\xa2\x86\x35\x16\xbf\x21\x38\x9f\x8d\x1e\x4f\xc9\x05\x7d\xf7\xfd\x82\xf2\xdf\x8e\xad\x38\xe9\xd7\x69\x36\xe0\xae\x47\x19\xf9\x02\x23\xbd\x21\xad\x5c\x4c\xcf\x63\x05\x43\x89\xc2\x24\xe9\x55\xff\x63\xf4\x90\x6b\xf4\xdc\xe6\x2f\x61\x13\x0d\x01\x35\xca\xca\xf4\x53\x48\xf7\x2d\xa8\x45\xa5\xa3\x24\x22\xa3\xd5\xe9\x95\xe1\xfa\xe8\x3f\x65\xd0\x00\x3d\x32\xac\x87\xee\x0f\x85\xc1\xbb\xe9\x1c\xb8\x38\x55\x32\xc5\xa5\x36\xb7\xac\xa5\x06\xe5\x0c\xf2\x18\x62\xe2\xac\x6f\xd2\x93\xce\x36\x4a\xcb\x8a\xdd\xa2\x30\x4d\x43\xa1\x55\xb1\x34\xb7\x60\x8a\x44\x35\x78\x24\xf1\x1b\xde\xe1\x76\x37\xd3\x04\x46\x2f\x31\x82\x3b\x3c\x39\x37\xc0\xfd\xab\xaf\x18\xe5\x3d\xa9\x5c\xca\xbe\x0a\x18\xd0\x4e\xdc\xf3\xfa\x77\xbc\xda\x4b\x9c\x1b\xde\x01\xa3\x0f\xcf\x8a\xd6\x4b\x47\x81\xae\x5f\xfc\x2f\x00\x00\xff\xff\x85\x31\xa0\xd8\x21\x2c\x00\x00") func templatesAlertgroupHtmlBytes() ([]byte, error) { return bindataRead( @@ -298,7 +298,7 @@ func templatesSummaryHtml() (*asset, error) { return a, nil } -var _staticAlertsJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x58\x6d\x6f\xdc\xb8\xf1\x7f\x9f\x4f\x31\xa7\x7f\x70\xd6\x26\xb2\xb4\x41\xf0\x4f\x8b\x75\x37\x80\x9b\x43\x03\xe3\x72\x0d\x60\xc7\x7d\xe3\x06\x05\x2d\xce\xae\x18\x73\x49\x81\x1c\x79\xbd\x3d\xf8\xbb\x17\x7c\xd0\xea\x71\xef\x72\x7d\xd1\x7d\x61\x4b\xe2\x3c\xfc\x66\x38\x9c\x07\x16\xaf\x60\x2b\xf5\x3d\x93\x16\x3e\x5d\xdf\xfe\xc2\x6a\x78\x55\x80\xfb\x15\x05\x48\xd3\xe4\xdf\xec\x8b\x1e\xc9\x4e\xef\x50\x51\x8f\x24\x7c\x70\x54\x7d\xb2\xcb\x86\x74\xa9\x77\xb5\x44\xc2\x0c\x3e\x68\xa9\x8d\x75\xff\xd5\x46\x6c\xdd\xff\x46\x11\x9a\x0c\x3e\x1a\xc1\x33\xf8\x82\xbb\x5a\x32\x42\x9b\xc1\x4d\xb3\xdb\x31\x73\xc8\xe0\xf6\x2a\x83\x5b\x65\x11\xe1\x55\xe1\x25\xe3\x53\xad\x0d\x21\x87\x4b\x89\x86\xac\xfb\xfc\xc8\x4c\xfb\xb6\x86\x74\xd3\xa8\x92\x84\x56\xe9\x02\x7e\x7d\xf1\xc2\xa1\x73\xeb\x56\x48\x54\x25\x3a\x8a\x5f\x9f\x33\xff\xd9\xfd\x24\xbb\x47\xf9\x81\x95\x15\xc2\x1a\x14\xee\xa3\xe9\xe9\x9b\xe5\x72\xb9\xb8\x08\xec\xad\xc0\xa0\xe3\xa3\xd1\x4d\x9d\x6e\xdd\xdf\x9f\x18\x31\xa7\xa4\x15\xf6\x32\xc7\x27\x42\xc5\x53\xaa\x84\xcd\xa0\xa3\xb9\xf0\x24\xcf\x41\x5c\x27\x25\xaf\x8d\x26\x4d\x87\x1a\xf3\x6b\x54\x1c\x0d\xac\x61\x80\xbe\x15\x6c\x90\x1a\xa3\x3a\xff\x44\xf2\x34\x61\x47\x59\x49\xd6\xa3\x77\x3f\xaf\x7d\x05\x1e\xca\x60\xa1\xf5\xc4\xea\xf8\x34\x5c\xf7\x32\x3f\x89\x9d\xa0\x15\xfc\xff\x71\xe5\xb9\x35\x22\x3a\xa5\x28\xa0\x64\x52\x22\x07\xb6\x21\x34\x41\x1d\xec\x99\x05\xe3\xb1\x21\x87\x8d\x36\x40\x15\xc2\x46\x18\x4b\x40\x62\x87\xa7\xed\xbf\xe4\x1c\xf9\x29\xf3\x6f\xaf\xf2\x1b\xa4\xa6\xee\x38\x6f\xaf\xd2\x97\x69\xf2\x7f\x09\xbc\xf6\x06\xe6\x82\x2f\x46\xf8\x66\xb5\xdc\xd6\x9c\x11\x9e\x52\x53\x14\x50\x09\x8e\x50\xeb\x5a\x3f\xa2\xb1\x20\x94\x97\x1e\x6c\xeb\x76\x79\xa8\x18\x5e\x43\x02\x77\x9c\x11\x3b\xf7\xc1\x74\xee\x34\xad\xcf\x36\x42\x12\x9a\xb3\xaf\xc9\x22\x8f\xf2\xd2\xc4\x49\x4f\xda\xa8\x8a\x1a\x0d\xee\xf4\x23\x02\x93\x12\x50\xa2\x3b\x42\x16\x6a\x23\x9c\xeb\x34\x94\x5a\x91\x3b\x65\x4d\xc0\x4d\x1a\xea\xc6\x6c\x23\xf5\xa3\x5b\x91\xc2\x12\x2a\x07\x96\x29\x0e\x95\xd6\x0f\xb6\x1f\x8e\xac\xac\x26\x8e\xca\x37\x42\xf1\x34\xc9\x6b\xa6\x50\x9e\xdf\x6b\x7e\xc8\x20\xbe\x54\xc8\xb8\x50\xdb\x64\x91\x75\x1e\x12\x99\x07\xb6\x18\x85\xd7\xcb\xd4\x7f\xcd\x03\xfe\x34\x7a\xbf\x8d\x94\x53\xce\x5a\xe4\x15\xed\x64\xfa\xd2\x9f\x90\x36\x8a\x17\xf1\xeb\xa2\x27\x64\xc2\xe7\x1c\x9c\x26\x15\xb3\x55\x92\x85\xaf\xee\x79\xe4\xcc\xba\x91\x16\x7d\xcc\xdd\x33\xbe\xf5\x0e\xb3\x95\xde\x03\x55\x8c\x62\x84\xb6\x2e\x75\x91\x5a\x56\x4c\x6d\x91\x67\x60\xb0\x46\x46\x20\x08\x68\x2f\x4a\xfc\xcd\xad\x7e\x1f\x9d\x75\x7c\x68\xbd\xe6\x3e\x78\xb5\x2b\xa1\xce\x1f\x05\xee\x5d\x9e\x4a\xbc\xbb\x85\xad\xd2\x45\xbe\x61\x1c\x3f\x37\x94\xbe\x5d\x2e\xc3\xcb\x95\xea\x9e\xe7\x16\xba\x88\x3e\xa6\x31\x8e\x96\x8c\x3e\xf8\xc0\xee\xc7\xb1\x37\xee\xea\xa7\x41\x36\x8a\xe0\xe3\xd2\x7f\x11\xa7\xbf\x2f\x89\xf4\x76\x2b\x71\x7d\x46\x5a\x4b\x12\xb5\x17\x13\x9f\x67\xc4\x0c\xe3\xb1\x85\xfc\xbf\x88\xc7\xf6\xd9\xd5\x99\xfc\x3a\x90\x8c\x71\x8c\x12\x88\x2f\x1a\xda\xd0\x2f\xac\xfe\xeb\xe1\x67\x3c\xf4\xbd\xbd\x63\xf5\x17\x7d\xa3\x0d\xf5\x51\x38\x86\x07\x3c\xb8\x0a\xf3\xf9\xfe\x1b\x96\x94\xbb\xb7\x1e\x6d\x07\xc3\x2d\xe4\x4e\x78\x1f\x67\xab\xd0\xe7\xc1\xbb\xaf\x13\xbf\x39\xa6\xa1\x1f\x1e\xf0\x30\x76\x43\x10\x90\xd7\x8d\xad\xd2\xe1\x4a\xd4\xbb\x72\x7f\xb2\xc9\xca\x23\x93\x0d\xae\xe0\x08\xf6\xee\x01\x0f\x5f\xa7\x64\x84\x4f\xe4\x25\xb8\x18\x58\x81\x73\xdf\x90\x65\xc0\xf1\x7c\x62\x17\x62\x39\x0b\x58\xa7\x5e\xf7\xe1\x79\x49\x64\x6c\xdf\xe7\x0e\x76\x80\x39\x76\xba\xa7\x87\xf5\x10\x96\xa7\xec\x65\x07\x47\xc8\xa2\xcc\xae\xe8\xe7\x5b\xa4\xd4\xbf\xf6\xe0\x89\x0d\xa4\x81\xf4\x87\xf5\x1a\x1a\xc5\x71\x23\x14\xf2\x45\x8b\xdb\xaf\xf5\x44\xb7\x62\x87\xfe\x0e\xae\xf2\xb2\x87\x7e\x2c\x25\xb3\x76\x05\x49\x80\x1d\xce\xa2\x4b\xe3\x1e\x76\xe8\x90\xf2\x8f\x48\x1f\x1c\x59\xdf\xea\x51\x15\xa7\x83\xc4\x55\x8f\xbe\x4f\xda\x79\xfd\x62\xa6\xd3\xc9\x6d\x6b\x74\x16\xb0\x4f\xb7\x26\x9a\x38\xde\x99\xaa\xd9\x31\x25\xfe\x8d\x5f\xc4\x0e\x2d\xb1\x5d\x6d\x4f\xd5\x52\x47\xad\xf4\x1e\xd6\xb1\x2f\xec\x07\xba\x6b\x1d\x7c\xe6\xf5\x3d\x81\x97\x13\xd0\x59\x97\xb0\xef\x11\x0c\x4a\x46\xe2\x11\x67\x32\x47\x1e\x73\x97\xfd\x9e\x9c\xe0\x40\xf8\x9e\x30\x62\x68\x73\x44\xa8\x25\x41\x46\x6c\x5b\xaf\x6e\x3e\xff\xeb\xcf\xef\x96\x6f\x7a\x30\xc7\xe1\x45\x36\xdf\x18\xbd\xfb\xbb\xde\xa7\x23\xaa\x56\x6e\x4c\x64\x2d\xc4\x73\x5b\x33\xe5\x12\x22\x3e\x4d\x83\xac\xcf\xe7\xbc\x9d\x26\x21\xa1\xda\x73\x12\x24\xd1\x15\x39\x9b\x93\xbe\x21\x23\xd4\x76\x50\x1a\x3b\xcb\x2e\xb7\xbe\x6b\xd5\xfb\x9c\x8b\xcd\x26\x25\x9b\x41\xb2\x13\xaa\x21\xb4\xc9\x88\xc1\xc5\x74\x60\x78\xbf\x86\x25\xfc\xf8\x63\x64\xff\x0b\xbc\x1d\x7b\x6d\x80\x8c\xf3\x10\x87\x89\xc1\x12\x15\x9d\xfb\xe6\x30\x39\x9a\x2a\x54\x29\xb8\xfb\x2e\x14\x17\x25\x23\x6d\x92\x36\x05\x47\xbe\x4a\x70\x8e\x6a\x0c\xe7\x19\xd0\x55\xea\xd3\x8a\x07\x32\xfe\x80\xee\x0e\xf0\x09\xc5\xf3\x6d\x4a\x51\xc0\x46\x32\x5b\x41\x50\x15\x7a\x60\xdb\x2f\x80\x79\x1f\xc4\x6f\x55\xf8\x2f\xbe\x24\x76\xb5\xfc\x0f\xbc\x4f\x0f\x5c\x68\xfc\x8e\xb3\xcd\x31\xe0\x59\x2d\xae\xd1\xd6\x5a\xd9\x49\x36\xf4\x00\xfd\x60\x05\x6b\x58\x0e\xcb\x8b\x2f\x75\x61\x04\x1a\x1a\x1f\x1b\xcc\x30\xb5\x75\xb3\x12\x17\x25\x01\xb3\x20\xe8\xcc\x82\x42\xe4\xb1\xab\x0f\x2d\xbe\x50\xdb\xa3\x8c\xde\x78\xd5\xc3\x96\xb7\x9f\x47\x89\xd8\x86\xe9\xce\xcd\x46\x11\xcc\xd1\xd3\xe1\xa4\xf7\x65\x94\x61\x48\xec\xd7\x3e\x7f\x98\x7e\x76\xf9\xae\x5d\x5c\x8c\xba\x81\x20\xe6\x14\xeb\x3f\x98\xcc\xa0\x12\x34\x66\xf3\x96\x74\xd8\xee\x5a\x3d\x5d\x59\x69\xd9\xbf\xc2\xda\x0b\xb8\xf8\xae\x82\x17\xa7\xd9\x38\x7f\xa4\x3d\x15\x83\x5e\x79\x6a\x7b\xd8\xb0\x61\xa6\x9b\x1d\x3d\x07\xbb\xdf\x76\x87\x6e\x9e\x9d\x9d\x59\x2f\xa6\x93\xa2\xbd\xeb\x58\x73\xc1\x9d\x7d\xdd\x87\x8b\xe9\x80\x18\x22\xec\x75\x9f\x2a\x0f\xa7\x26\x97\xa8\xb6\x54\x9d\x18\x08\xe2\xc8\xef\x06\xba\xb4\x13\x34\x6e\xd1\x6e\xaa\x90\x65\x07\x51\xc3\x85\x21\xdf\x86\x31\x69\x71\x18\xbf\x15\x53\x5c\xba\xb9\xc8\x20\xe3\x07\xc0\x27\x61\xc9\xb5\xe4\xc1\xb4\xb1\x87\xbd\x8a\x2b\xc2\x9d\x4d\xc7\x55\x24\x32\x7a\x7b\xe6\xfc\xbb\x8d\xae\x8d\x3e\x1b\xd0\x3b\xb7\x4d\xb3\x6e\xe0\x18\x75\x12\xd3\xb0\x2b\x8a\x28\xdb\x92\x90\x12\x6a\x83\x16\x15\x65\x50\x56\x58\x3e\x38\x41\x71\x58\x99\x30\x1e\x75\xf8\x81\x08\x7e\x58\x0f\x8d\xf0\xc5\xce\x22\x85\x71\x69\x46\xf1\x40\xb9\x9b\x8a\x42\x2e\xf0\x53\x91\xbf\x93\x08\x8a\xed\x2c\x63\x50\x1c\xc3\x7a\x14\x56\xed\xef\x34\x9c\xd6\x8f\xfe\x65\x9e\xb9\xdd\x73\x32\x83\x6e\x6e\x04\xbe\x22\xaa\xed\xaa\x28\xb6\x82\xaa\xe6\x3e\x2f\xf5\xae\xa0\xfd\xbd\x2d\xee\xb5\x26\x4b\x86\xd5\x85\xb0\xb6\x41\x5b\xbc\x79\xf7\xf6\x4f\xef\x66\xa5\x58\x24\xd7\xd5\xe8\x86\xd2\xd9\x86\x66\xde\x72\x7f\x6d\x71\xca\xf0\xe7\x0c\xe2\x35\xd2\x64\xe5\xfb\x8a\xe1\x71\x5b\x84\x85\xad\x56\x98\xb5\xe3\x1f\x08\x9a\x10\xf7\x27\xc3\x74\x1c\x97\x33\x18\x86\x9e\x3d\x8d\xae\xbb\xa3\x70\xdd\x4f\x5b\x42\xda\xca\xb0\xc7\x33\x83\xc0\xb5\x42\xd8\x0b\xaa\xc6\xc0\x38\x4a\x74\x75\xe5\x77\x4f\xcb\xb8\x1a\xc7\xd8\x73\x19\x6c\x74\x86\xdd\x29\x6c\x87\xf8\xd9\x39\x69\x9a\x33\x79\x4c\x9a\xe3\xfd\x8c\x62\xc2\xb4\x14\x36\xf4\x78\x25\x31\x9f\xc8\x8b\x02\x58\x5d\xa3\xe2\x3d\x64\x20\x14\x94\x55\xa3\x7a\x37\x2f\xee\x4c\xb6\xc2\x43\x36\x84\xf7\xb0\x1c\xab\xf7\x69\xe8\xd2\x8b\x4b\x5f\x1e\xe9\x6d\x2d\x45\x89\xe9\xd2\x07\xcf\x22\xff\xa6\x85\x4a\x93\x7f\xaa\x64\x31\xee\x01\xe7\x37\xf0\x79\xe0\x46\x26\xf7\xec\x60\xc1\xe0\xc6\xa0\xad\x26\xfd\xf6\x91\x74\xda\xd8\xa7\x33\x95\xe9\xfb\x3d\x7b\xe2\x74\x0c\x76\xd9\x39\xc9\xdb\x30\x66\xee\xdf\x19\xe7\xd7\xe8\xa6\x95\x91\xe9\x71\x8c\xe7\x86\x4d\x7a\x71\x27\x35\x5c\x2f\xbb\xa1\xe8\x73\xed\x71\x26\xbe\xb9\x4b\x16\x7e\x4e\x5a\xcc\x1d\x6b\x7f\xc5\x9c\xff\xcd\x91\x8d\x25\x3e\x8f\x7d\xdb\xf6\x4e\x71\x56\xea\xdd\x53\xfa\x3c\xb8\x1a\x74\x6d\xdd\xd4\x76\xd3\xbb\x4c\x58\x0d\xae\x16\x3a\x9a\x8f\x48\x9f\x8e\xd3\xef\xaa\x37\x09\x1f\x15\x3f\xfb\xd8\xfc\x4f\x00\x00\x00\xff\xff\x32\xcf\x7a\xe4\xac\x17\x00\x00") +var _staticAlertsJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x58\xdd\x6f\xdc\xb8\x11\x7f\xcf\x5f\x31\xa7\x06\x17\x6d\x22\x4b\x1b\x04\x4d\x8b\x75\x37\x80\x9b\x43\x03\xe3\x72\x0d\x60\xc7\x7d\x71\x83\x82\x16\x67\x57\x8c\xb5\xa4\x40\x8e\xbc\xde\x16\xfe\xdf\x0b\x7e\x68\x45\x7d\xec\xd5\xd7\x87\xdb\x07\x5b\x12\xe7\xe3\x37\xc3\xe1\x7c\xb0\x78\x0d\xdb\x5a\xdd\xb1\xda\xc0\xe7\xab\x9b\x5f\x58\x03\xaf\x0b\xb0\xbf\xa2\x80\x5a\xb7\xf9\x77\xf3\x22\x22\xd9\xa9\x1d\x4a\x8a\x48\xfc\x07\x4b\x15\x93\x5d\xb4\xa4\x4a\xb5\x6b\x6a\x24\xcc\xe0\xa3\xaa\x95\x36\xf6\xbf\xdc\x88\xad\xfd\xdf\x4a\x42\x9d\xc1\x27\x2d\x78\x06\x5f\x71\xd7\xd4\x8c\xd0\x64\x70\xdd\xee\x76\x4c\x1f\x32\xb8\xb9\xcc\xe0\x46\x1a\x44\x78\x5d\x38\xc9\xf8\xd8\x28\x4d\xc8\xe1\xa2\x46\x4d\xc6\x7e\x7e\x60\xba\x7b\x5b\x43\xba\x69\x65\x49\x42\xc9\x74\x01\xff\x79\xf1\xc2\xa2\xb3\xeb\x35\xbb\xc3\xfa\x23\x2b\x2b\x84\x35\x48\xdc\x07\x1b\xd3\xb7\xcb\xe5\x72\x71\xee\xe9\x3a\x4e\x2f\xec\x93\x56\x6d\x93\x6e\xed\xdf\x9f\x18\x31\x2b\x0d\xc2\xef\x65\x8e\x8f\x84\x92\xa7\x54\x09\x93\x41\x4f\x73\xee\x48\x9e\xbc\xb8\x5e\x4a\xde\x68\x45\x8a\x0e\x0d\xe6\x57\x28\x39\x6a\x58\xc3\x00\x66\x27\x58\x23\xb5\x5a\xf6\x8e\x08\xe4\x69\xc2\x8e\xb2\x92\x2c\xa2\xb7\x3f\xa7\x7d\x05\x0e\xca\x60\xc1\xf1\x7c\x16\x3b\x41\x2b\xf8\xe3\x71\xe5\xa9\x03\x19\x8c\x2e\x0a\x28\x59\x5d\x23\x07\xb6\x21\xd4\x5e\x1c\xec\x99\x01\xed\x74\x23\x87\x8d\xd2\x40\x15\xc2\x46\x68\x43\x40\x62\x87\xa7\xed\xbb\xe0\x1c\xf9\x29\xf3\x6e\x2e\xf3\x6b\xa4\xb6\xe9\x39\x6f\x2e\xd3\x97\x69\xf2\x87\x04\xde\x38\x03\x72\xc1\x17\x23\x7c\xb3\x5a\x6e\x1a\xce\x08\x4f\xa9\x29\x0a\xa8\x04\x47\x68\x54\xa3\x1e\x50\x1b\x10\xd2\x49\xf7\xb6\xf5\xbb\x38\x54\x0c\x6f\x20\x81\x5b\xce\x88\x9d\xb9\x60\x39\xb3\x9a\xd6\xaf\x36\xa2\x26\xd4\xaf\xbe\x25\x8b\x3c\xc8\x4b\x13\x2b\x3d\xe9\xa2\x26\x68\xd4\xb8\x53\x0f\x08\xac\xae\x01\x6b\xb4\x67\xc1\x40\xa3\x85\x75\x9d\x82\x52\x49\xb2\xc7\xa5\xf5\xb8\x49\x41\xd3\xea\x6d\xa0\x7e\xb0\x2b\xb5\x30\x84\xd2\x82\x65\x92\x43\xa5\xd4\xbd\x89\xc3\x8d\x95\xd5\xc4\x51\xf9\x46\x48\x9e\x26\x79\xc3\x24\xd6\x67\x77\x8a\x1f\x32\x08\x2f\x15\x32\x2e\xe4\x36\x59\x64\xbd\x87\x44\xe6\x80\x2d\x46\xe1\xf3\x32\x75\x5f\x73\x8f\x3f\x0d\xde\xef\x22\xe5\x94\xb3\x16\x79\x45\xbb\x3a\x7d\xe9\x4e\x40\x17\xa5\x8b\xf0\x75\x11\x09\x99\xf0\x59\x07\xa7\x49\xc5\x4c\x95\x64\xfe\xab\x7d\x1e\x39\xb3\x69\x6b\x83\x2e\xe6\xee\x18\xdf\x3a\x87\x99\x4a\xed\x81\x2a\x46\x21\x42\x3b\x97\xda\x48\x2d\x2b\x26\xb7\xc8\x33\xd0\xd8\x20\x23\x10\x04\xb4\x17\x25\xfe\xea\x56\x7f\x08\xce\x3a\x3e\x74\x5e\xb3\x1f\x9c\xda\x95\x90\x67\x0f\x02\xf7\x36\xe1\x24\xce\xdd\xc2\x54\xe9\x22\xdf\x30\x8e\x5f\x5a\x4a\xdf\x2d\x97\xfe\xe5\x52\xf6\xcf\x73\x0b\x7d\x44\x1f\xf3\x11\x47\x43\x5a\x1d\x5c\x60\xc7\x71\xec\x8c\xbb\xfc\x69\x90\x6d\x02\xf8\xb0\xf4\x7f\xc4\xe9\xff\x96\x44\x6a\xbb\xad\x71\xfd\x8a\x94\xaa\x49\x34\x4e\x4c\x78\x9e\x11\x33\x8c\xc7\x0e\xf2\xef\x11\x8f\xdd\xb3\x2d\x18\xf9\x95\x27\x19\xe3\x18\x25\x10\xeb\x6d\xa3\x34\xfd\xc2\x9a\xbf\x1e\x7e\xc6\x43\xec\xed\x1d\x6b\xbe\xaa\x6b\xa5\x29\x46\x61\x19\xee\xf1\x60\x8b\xc9\x97\xbb\xef\x58\x52\x6e\xdf\x22\xda\x1e\x86\x5d\xc8\xad\xf0\x18\x67\xa7\xd0\xe5\xc1\xdb\x6f\x13\xbf\x59\xa6\xa1\x1f\xee\xf1\x30\x76\x83\x17\x90\x37\xad\xa9\xd2\xe1\x4a\xd0\xbb\xb2\x7f\xb2\xc9\xca\x03\xab\x5b\x5c\xc1\x11\xec\xed\x3d\x1e\xbe\x4d\xc9\x08\x1f\xc9\x49\xb0\x31\xb0\x02\xeb\xbe\x21\xcb\x80\xe3\xe9\xc4\x2e\x84\x72\xe5\xb1\x4e\xbd\xee\xc2\xf3\x82\x48\x9b\xd8\xe7\x16\xb6\x87\x39\x76\xba\xa3\x87\xf5\x10\x96\xa3\x8c\xb2\x83\x25\x64\x41\x66\x5f\xd4\xf3\x2d\x52\xea\x5e\x23\x78\x62\x03\xa9\x27\xfd\x61\xbd\x86\x56\x72\xdc\x08\x89\x7c\xd1\xe1\x76\x6b\x91\xe8\x4e\xec\xd0\xdf\xde\x55\x4e\xf6\xd0\x8f\x65\xcd\x8c\x59\x41\xe2\x61\xfb\xb3\x68\xd3\xb8\x83\xed\x5b\x9d\xfc\x13\xd2\x47\x4b\x16\x5b\x3d\x94\x62\xe8\x50\xe3\x2a\xa2\x8f\x49\x7b\xaf\xf7\x56\x45\x46\x9b\xce\xe8\xcc\x63\x9f\x6e\x4d\x30\x71\xbc\x33\x55\xbb\x63\x52\xfc\x1b\xbf\x8a\x1d\x1a\x62\xbb\xc6\x9c\xaa\xa5\x96\x5a\xaa\x3d\xac\x43\x83\x17\x07\xba\x6d\x1d\x5c\xe6\x75\x3d\x81\x93\xe3\xd1\x19\x9b\xb0\xef\x10\x34\xd6\x8c\xc4\x03\xce\x64\x8e\x3c\xe4\x2e\xf3\x9c\x9c\x60\x41\xb8\xe6\x2e\x60\xe8\x72\x84\xaf\x25\x5e\x46\xe8\x3f\x2f\xaf\xbf\xfc\xeb\xcf\xef\x97\x6f\x23\x98\xe3\xf0\x22\x93\x6f\xb4\xda\xfd\x5d\xed\xd3\x11\x55\x27\x37\x24\xb2\x0e\xe2\x99\x69\x98\xb4\x09\x11\x1f\xa7\x41\x16\xf3\x59\x6f\xa7\x89\x4f\xa8\xe6\x8c\x04\xd5\x68\x8b\x9c\xc9\x49\x5d\x93\x16\x72\x3b\x28\x8d\xbd\x65\x17\x5b\xd7\x95\xaa\x7d\xce\xc5\x66\x93\x92\xc9\x20\xd9\x09\xd9\x12\x9a\x64\xc4\x60\x63\xda\x33\x7c\x58\xc3\x12\x7e\xfc\x31\xb0\xff\x05\xde\x8d\xbd\x36\x40\xc6\xb9\x8f\xc3\x44\x63\x89\x92\xce\x5c\x73\x98\x1c\x4d\x15\xb2\x14\xdc\x7e\x17\x92\x8b\x92\x91\xd2\x49\x97\x82\x03\x5f\x25\x38\x47\x39\x86\xf3\x04\x68\x2b\xf5\x69\xc5\x03\x19\xbf\x41\x77\x0f\xf8\x84\xe2\xf9\x36\xa5\x28\x60\x53\x33\x53\x81\x57\xe5\x7b\x60\x13\x17\xc0\x3c\x06\xf1\x6b\x15\xfe\xab\x2b\x89\x7d\x2d\xff\x0d\xef\xd3\x03\xe7\x1b\xbf\xe3\x90\x72\x0c\x78\xd6\x88\x2b\x34\x8d\x92\x66\x92\x0d\x1d\x40\x37\x21\xc1\x1a\x96\xc3\xf2\xe2\x4a\x9d\x4b\x55\x4f\xa3\xd4\x68\xfc\xe0\x64\xa7\x91\xb0\x7c\xb4\xdd\x9f\xbd\x48\x63\x5e\xfa\xf9\x2b\xae\x46\x2e\xbc\x7f\xb6\x19\xa8\x5b\x5c\x8c\xea\xb3\x17\x73\x8a\xf5\x1f\xac\xce\xa0\x12\x34\x66\xb3\xbf\x08\xdb\x6d\xa7\xa7\x4f\xf4\x1d\xfb\x37\x58\x3b\x01\xe7\xcf\x2a\x41\x61\x50\x0c\x13\x41\x1a\xa9\x18\x74\xaf\x53\xdb\xbd\x0b\x87\xb9\x67\x76\xd8\x1b\xec\x47\xd7\xaf\xd9\x09\x72\x76\x4a\x3c\x9f\xce\x66\xe6\xb6\x67\xcd\x05\xb7\xf6\xf5\x1f\xce\xa7\x23\x9b\xdf\xf3\x37\x31\x55\xee\xe3\x38\xaf\x51\x6e\xa9\x3a\xd1\xa2\x87\x69\xda\x8e\x58\x69\x2f\x68\xdc\x34\x5d\x57\x3e\xef\x0d\xa2\x86\x0b\x4d\xae\x31\x62\xb5\xc1\xe1\x71\xaa\x98\xe4\xb5\x9d\x54\x34\x32\x7e\x00\x7c\x14\x86\x6c\x93\xec\x4d\x1b\x7b\xd8\xa9\xb8\x24\xdc\x99\x74\x9c\xd7\x03\xa3\xb3\x67\xce\xbf\xdb\xe0\xda\xe0\xb3\x01\xbd\x75\xdb\x34\x0f\x7a\x8e\x51\x6d\x9f\x86\x5d\x51\x04\xd9\x86\x44\x5d\x43\xa3\xd1\xa0\xa4\x0c\xca\x0a\xcb\x7b\x2b\x28\x8c\x0f\x13\xc6\xa3\x0e\x37\xa2\xc0\x0f\xeb\xa1\x11\xae\xfc\x18\x24\x3f\xc0\xcc\x28\x1e\x28\xb7\x73\x8a\xcf\x02\x6e\x4e\x71\xb7\x00\x5e\xb1\x99\x65\xf4\x8a\x43\x58\x8f\xc2\xaa\xfb\x9d\x86\xd3\xf9\xd1\xbd\xcc\x33\x77\x7b\x4e\x7a\xd0\x5f\x8d\xc0\x57\x44\x8d\x59\x15\xc5\x56\x50\xd5\xde\xe5\xa5\xda\x15\xb4\xbf\x33\xc5\x9d\x52\x64\x48\xb3\xa6\x10\xc6\xb4\x68\x8a\xb7\xef\xdf\xfd\xe9\xfd\xac\x14\x83\x64\xfb\x0c\xd5\x52\x3a\xdb\x62\xcc\x5b\xee\x2e\x12\x4e\x19\xfe\x94\x41\xb8\xb8\x99\xac\x3c\xaf\x3c\x1d\xb7\x45\x18\xd8\x2a\x89\x59\x37\x90\x81\xa0\x09\x71\x3c\xab\xa5\xe3\xb8\x9c\xc1\x30\xf4\xec\x69\x74\xfd\xad\x81\xed\x47\xba\xa4\xce\x45\x49\xc0\x0c\xec\xf1\x95\x46\xe0\x4a\x22\xec\x05\x55\x63\x60\x1c\x6b\x24\x7c\xc6\x69\x19\xd7\xc7\x10\x7b\x36\x83\x8d\xce\xb0\x3d\x85\xdd\x58\x3d\x3b\xb9\x4c\x73\x26\x0f\x49\x73\xbc\x9f\x41\x8c\x9f\x5f\xfc\x86\x1e\x2f\x09\xe6\x13\x79\x51\x00\x6b\x1a\x94\x3c\x42\x06\x42\x42\x59\xb5\x32\xba\x0b\xb1\x67\xb2\x13\xee\xb3\x21\x7c\x80\xe5\x58\xbd\x4b\x43\x17\x4e\x5c\xfa\xf2\x48\x6f\x9a\x5a\x94\x98\x2e\x5d\xf0\x2c\xf2\xef\x4a\xc8\x34\xf9\xa7\x4c\x16\xe3\xae\x6c\x7e\x03\x9f\x06\x6e\x64\xf5\x9e\x1d\x0c\x68\xdc\x68\x34\xd5\xa4\x03\x3e\x92\x4e\x5b\xed\x74\xa6\x32\x3d\xdf\xb3\x27\x4e\xc7\x60\x97\xad\x93\x9c\x0d\x63\xe6\xf8\x3a\x36\xbf\x42\x3b\x3f\x8c\x4c\x0f\x83\x35\xd7\x6c\xd2\x1d\x5b\xa9\xfe\xe6\xd6\x8e\x29\x5f\x1a\x87\x33\x71\xed\x56\xb2\x70\x93\xcb\x62\xee\x58\xbb\xdb\xdb\xfc\x6f\x96\x6c\x2c\xf1\x69\xec\xdb\xae\x9b\x09\xd3\x4b\x74\x73\xe8\xf2\xe0\x6a\xd0\x47\xf5\x73\xd4\x75\x34\xde\xaf\x06\xc3\x7e\x4f\xf3\x09\xe9\xf3\x71\x1e\x5d\x45\xb3\xe9\x51\xf1\x93\x8b\xcd\xff\x06\x00\x00\xff\xff\xb3\x24\x3c\x79\x07\x17\x00\x00") func staticAlertsJsBytes() ([]byte, error) { return bindataRead( diff --git a/config/config.go b/config/config.go index e22941574..621f13e6b 100644 --- a/config/config.go +++ b/config/config.go @@ -25,7 +25,7 @@ func (mvd *spaceSeparatedList) Decode(value string) error { type configEnvs struct { AlertmanagerTimeout time.Duration `envconfig:"ALERTMANAGER_TIMEOUT" default:"40s" help:"Timeout for all request send to Alertmanager"` AlertmanagerTTL time.Duration `envconfig:"ALERTMANAGER_TTL" default:"1m" help:"TTL for Alertmanager alerts and silences"` - AlertmanagerURI string `envconfig:"ALERTMANAGER_URI" required:"true" help:"Alertmanager URI"` + AlertmanagerURI spaceSeparatedList `envconfig:"ALERTMANAGER_URI" required:"true" help:"Alertmanager URIs"` ColorLabelsStatic spaceSeparatedList `envconfig:"COLOR_LABELS_STATIC" help:"List of label names that should have the same (but distinct) color"` ColorLabelsUnique spaceSeparatedList `envconfig:"COLOR_LABELS_UNIQUE" help:"List of label names that should have unique color"` Debug bool `envconfig:"DEBUG" default:"false" help:"Enable debug mode"` diff --git a/filters/filter_fuzzy.go b/filters/filter_fuzzy.go index c2d7d19de..af7485a72 100644 --- a/filters/filter_fuzzy.go +++ b/filters/filter_fuzzy.go @@ -5,7 +5,6 @@ import ( "regexp" "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/store" ) type fuzzyFilter struct { @@ -42,10 +41,14 @@ func (filter *fuzzyFilter) Match(alert *models.Alert, matches int) bool { } for _, silenceID := range alert.SilencedBy { - if silence := store.Store.GetSilence(silenceID); silence != nil { - if filter.Matcher.Compare(silence.Comment, filter.Value) { - filter.Hits++ - return true + for _, am := range alert.Alertmanager { + silence, found := am.Silences[silenceID] + if found { + m := filter.Matcher.Compare(silence.Comment, filter.Value) + if m { + filter.Hits++ + return true + } } } } diff --git a/filters/filter_silence_author.go b/filters/filter_silence_author.go index b4e0928cc..d39cf10e0 100644 --- a/filters/filter_silence_author.go +++ b/filters/filter_silence_author.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/store" ) type silenceAuthorFilter struct { @@ -17,8 +16,14 @@ func (filter *silenceAuthorFilter) Match(alert *models.Alert, matches int) bool var isMatch bool if alert.IsSilenced() { for _, silenceID := range alert.SilencedBy { - if silence := store.Store.GetSilence(silenceID); silence != nil { - isMatch = filter.Matcher.Compare(filter.Value, silence.CreatedBy) + for _, am := range alert.Alertmanager { + silence, found := am.Silences[silenceID] + if found { + m := filter.Matcher.Compare(silence.CreatedBy, filter.Value) + if m { + isMatch = m + } + } } } } else { @@ -43,15 +48,18 @@ func sinceAuthorAutocomplete(name string, operators []string, alerts []models.Al for _, alert := range alerts { if alert.IsSilenced() { for _, silenceID := range alert.SilencedBy { - if silence := store.Store.GetSilence(silenceID); silence != nil { - for _, operator := range operators { - token := fmt.Sprintf("%s%s%s", name, operator, silence.CreatedBy) - tokens[token] = makeAC(token, []string{ - name, - strings.TrimPrefix(name, "@"), - fmt.Sprintf("%s%s", name, operator), - silence.CreatedBy, - }) + for _, am := range alert.Alertmanager { + silence, found := am.Silences[silenceID] + if found { + for _, operator := range operators { + token := fmt.Sprintf("%s%s%s", name, operator, silence.CreatedBy) + tokens[token] = makeAC(token, []string{ + name, + strings.TrimPrefix(name, "@"), + fmt.Sprintf("%s%s", name, operator), + silence.CreatedBy, + }) + } } } } diff --git a/filters/filter_silence_jira.go b/filters/filter_silence_jira.go index b76f5b1c5..8f62f37d5 100644 --- a/filters/filter_silence_jira.go +++ b/filters/filter_silence_jira.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/store" ) type silenceJiraFilter struct { @@ -17,8 +16,14 @@ func (filter *silenceJiraFilter) Match(alert *models.Alert, matches int) bool { var isMatch bool if alert.IsSilenced() { for _, silenceID := range alert.SilencedBy { - if silence := store.Store.GetSilence(silenceID); silence != nil { - isMatch = filter.Matcher.Compare(silence.JiraID, filter.Value) + for _, am := range alert.Alertmanager { + silence, found := am.Silences[silenceID] + if found { + m := filter.Matcher.Compare(silence.JiraID, filter.Value) + if m { + isMatch = m + } + } } } } else { @@ -43,16 +48,18 @@ func sinceJiraIDAutocomplete(name string, operators []string, alerts []models.Al for _, alert := range alerts { if alert.IsSilenced() { for _, silenceID := range alert.SilencedBy { - silence := store.Store.GetSilence(silenceID) - if silence != nil && silence.JiraID != "" { - for _, operator := range operators { - token := fmt.Sprintf("%s%s%s", name, operator, silence.JiraID) - tokens[token] = makeAC(token, []string{ - name, - strings.TrimPrefix(name, "@"), - fmt.Sprintf("%s%s", name, operator), - silence.JiraID, - }) + for _, am := range alert.Alertmanager { + silence, found := am.Silences[silenceID] + if found && silence.JiraID != "" { + for _, operator := range operators { + token := fmt.Sprintf("%s%s%s", name, operator, silence.JiraID) + tokens[token] = makeAC(token, []string{ + name, + strings.TrimPrefix(name, "@"), + fmt.Sprintf("%s%s", name, operator), + silence.JiraID, + }) + } } } } diff --git a/filters/filter_test.go b/filters/filter_test.go index 9374ba5ed..f1de300e0 100644 --- a/filters/filter_test.go +++ b/filters/filter_test.go @@ -5,9 +5,11 @@ import ( "testing" "time" + "github.com/cloudflare/unsee/alertmanager" "github.com/cloudflare/unsee/filters" "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/store" + + log "github.com/Sirupsen/logrus" ) type filterTest struct { @@ -433,14 +435,27 @@ var tests = []filterTest{ } func TestFilters(t *testing.T) { + log.SetLevel(log.ErrorLevel) + + err := alertmanager.NewAlertmanager("test", "http://localhost", time.Second) + am := alertmanager.GetAlertmanagerByName("test") + if err != nil { + t.Error(err) + } for _, ft := range tests { + alert := models.Alert(ft.Alert) if &ft.Silence != nil { - store.Store.SetSilences(map[string]models.Silence{ - ft.Silence.ID: ft.Silence, - }) - } else { - store.Store.SetSilences(map[string]models.Silence{}) + alert.Alertmanager = []models.AlertmanagerUpstream{ + models.AlertmanagerUpstream{ + Name: am.Name, + URI: am.URI, + Silences: map[string]models.Silence{ + ft.Silence.ID: ft.Silence, + }, + }, + } } + f := filters.NewFilter(ft.Expression) if f == nil { t.Errorf("[%s] No filter found", ft.Expression) @@ -452,7 +467,7 @@ func TestFilters(t *testing.T) { t.Errorf("[%s] GetIsValid() returned %#v while %#v was expected", ft.Expression, f.GetIsValid(), ft.IsValid) } if f.GetIsValid() { - m := f.Match(&ft.Alert, 0) + m := f.Match(&alert, 0) if m != ft.IsMatch { j, _ := json.Marshal(ft.Alert) s, _ := json.Marshal(ft.Silence) diff --git a/main.go b/main.go index b184f5e17..19ca65660 100644 --- a/main.go +++ b/main.go @@ -3,9 +3,9 @@ package main import ( "path" "strings" - "sync" "time" + "github.com/cloudflare/unsee/alertmanager" "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/transform" @@ -17,7 +17,6 @@ import ( "github.com/gin-gonic/gin" ginprometheus "github.com/mcuadros/go-gin-prometheus" "github.com/patrickmn/go-cache" - "github.com/prometheus/client_golang/prometheus" ) var ( @@ -31,46 +30,8 @@ var ( // If there are requests with the same filter we should respond from cache // rather than do all the filtering every time apiCache *cache.Cache - - // errorLock holds a mutex used to synchronize updates to AlertmanagerError - // to avoid any race between readers and writers - errorLock = sync.RWMutex{} - // alertManagerError holds the description of last error raised when pulling data - // from Alertmanager, if there was any error - // This error will be returned in UnseeAlertsResponse and presented by Ui - alertManagerError string - - metricAlerts = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "unsee_collected_alerts", - Help: "Total number of alerts collected from Alertmanager API", - }, - []string{"state"}, - ) - metricAlertGroups = prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "unsee_collected_groups", - Help: "Total number of alert groups collected from Alertmanager API", - }, - ) - metricAlertmanagerErrors = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "unsee_alertmanager_errors_total", - Help: "Total number of errors encounter when requesting data from Alertmanager API", - }, - []string{"endpoint"}, - ) ) -func init() { - prometheus.MustRegister(metricAlerts) - prometheus.MustRegister(metricAlertGroups) - prometheus.MustRegister(metricAlertmanagerErrors) - - metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "alerts"}).Set(0) - metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "silences"}).Set(0) -} - func getViewURL(sub string) string { u := path.Join(config.Config.WebPrefix, sub) if strings.HasSuffix(sub, "/") { @@ -91,6 +52,18 @@ func setupRouter(router *gin.Engine) { router.GET(getViewURL("/autocomplete.json"), autocomplete) } +func setupUpstreams() { + for _, s := range config.Config.AlertmanagerURI { + z := strings.SplitN(s, ":", 2) + if len(z) != 2 { + log.Fatalf("Invalid Alertmanager URI '%s', expected format 'name:uri'", s) + } + name := z[0] + uri := z[1] + alertmanager.NewAlertmanager(name, uri, config.Config.AlertmanagerTimeout) + } +} + func main() { log.Infof("Version: %s", version) @@ -100,9 +73,11 @@ func main() { apiCache = cache.New(cache.NoExpiration, 10*time.Second) + setupUpstreams() + // before we start try to fetch data from Alertmanager log.Infof("Initial Alertmanager query, this can delay startup up to %s", 3*config.Config.AlertmanagerTimeout) - PullFromAlertmanager() + pullFromAlertmanager() log.Info("Done, starting HTTP server") // background loop that will fetch updates from Alertmanager diff --git a/mapper/mapper.go b/mapper/mapper.go index 40258412b..13a1a47a7 100644 --- a/mapper/mapper.go +++ b/mapper/mapper.go @@ -2,6 +2,7 @@ package mapper import ( "fmt" + "time" "github.com/cloudflare/unsee/models" ) @@ -15,7 +16,7 @@ var ( // for a specific range of Alertmanager versions type AlertMapper interface { IsSupported(version string) bool - GetAlerts() ([]models.AlertGroup, error) + GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) } // RegisterAlertMapper allows to register mapper implementing alert data @@ -39,7 +40,7 @@ func GetAlertMapper(version string) (AlertMapper, error) { type SilenceMapper interface { Release() string IsSupported(version string) bool - GetSilences() ([]models.Silence, error) + GetSilences(uri string, timeout time.Duration) ([]models.Silence, error) } // RegisterSilenceMapper allows to register mapper implementing silence data diff --git a/mapper/v04/alerts.go b/mapper/v04/alerts.go index 54f7c18c8..6f4d577b3 100644 --- a/mapper/v04/alerts.go +++ b/mapper/v04/alerts.go @@ -10,7 +10,6 @@ import ( "time" "github.com/blang/semver" - "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/mapper" "github.com/cloudflare/unsee/models" "github.com/cloudflare/unsee/transport" @@ -60,17 +59,17 @@ func (m AlertMapper) IsSupported(version string) bool { // GetAlerts will make a request to Alertmanager API and parse the response // It will only return alerts or error (if any) -func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) { +func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) { groups := []models.AlertGroup{} receivers := map[string]alertsGroupReceiver{} resp := alertsGroupsAPISchema{} - url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups") + url, err := transport.JoinURL(uri, "api/v1/alerts/groups") if err != nil { return groups, err } - err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp) + err = transport.ReadJSON(url, timeout, &resp) if err != nil { return groups, err } diff --git a/mapper/v04/silences.go b/mapper/v04/silences.go index 239b1420e..c3fb5b5c8 100644 --- a/mapper/v04/silences.go +++ b/mapper/v04/silences.go @@ -12,7 +12,6 @@ import ( "time" "github.com/blang/semver" - "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/mapper" "github.com/cloudflare/unsee/models" "github.com/cloudflare/unsee/transport" @@ -56,18 +55,18 @@ func (m SilenceMapper) IsSupported(version string) bool { // GetSilences will make a request to Alertmanager API and parse the response // It will only return silences or error (if any) -func (m SilenceMapper) GetSilences() ([]models.Silence, error) { +func (m SilenceMapper) GetSilences(uri string, timeout time.Duration) ([]models.Silence, error) { silences := []models.Silence{} resp := silenceAPISchema{} - url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences") + url, err := transport.JoinURL(uri, "api/v1/silences") if err != nil { return silences, err } // Alertmanager 0.4 uses pagination for silences url = fmt.Sprintf("%s?limit=%d", url, math.MaxUint32) - err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp) + err = transport.ReadJSON(url, timeout, &resp) if err != nil { return silences, err } diff --git a/mapper/v05/alerts.go b/mapper/v05/alerts.go index 68db5fff9..586d602cf 100644 --- a/mapper/v05/alerts.go +++ b/mapper/v05/alerts.go @@ -9,7 +9,6 @@ import ( "time" "github.com/blang/semver" - "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/mapper" "github.com/cloudflare/unsee/models" "github.com/cloudflare/unsee/transport" @@ -59,17 +58,17 @@ func (m AlertMapper) IsSupported(version string) bool { // GetAlerts will make a request to Alertmanager API and parse the response // It will only return alerts or error (if any) -func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) { +func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) { groups := []models.AlertGroup{} receivers := map[string]alertsGroupReceiver{} resp := alertsGroupsAPISchema{} - url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups") + url, err := transport.JoinURL(uri, "api/v1/alerts/groups") if err != nil { return groups, err } - err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp) + err = transport.ReadJSON(url, timeout, &resp) if err != nil { return groups, err } diff --git a/mapper/v05/silences.go b/mapper/v05/silences.go index b2855ce14..cfd815c8c 100644 --- a/mapper/v05/silences.go +++ b/mapper/v05/silences.go @@ -9,7 +9,6 @@ import ( "time" "github.com/blang/semver" - "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/mapper" "github.com/cloudflare/unsee/models" "github.com/cloudflare/unsee/transport" @@ -48,16 +47,16 @@ func (m SilenceMapper) IsSupported(version string) bool { // GetSilences will make a request to Alertmanager API and parse the response // It will only return silences or error (if any) -func (m SilenceMapper) GetSilences() ([]models.Silence, error) { +func (m SilenceMapper) GetSilences(uri string, timeout time.Duration) ([]models.Silence, error) { silences := []models.Silence{} resp := silenceAPISchema{} - url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences") + url, err := transport.JoinURL(uri, "api/v1/silences") if err != nil { return silences, err } - err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp) + err = transport.ReadJSON(url, timeout, &resp) if err != nil { return silences, err } diff --git a/mapper/v061/alerts.go b/mapper/v061/alerts.go index 1b5100550..5b0f8f43d 100644 --- a/mapper/v061/alerts.go +++ b/mapper/v061/alerts.go @@ -10,7 +10,6 @@ import ( "time" "github.com/blang/semver" - "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/mapper" "github.com/cloudflare/unsee/models" "github.com/cloudflare/unsee/transport" @@ -61,17 +60,17 @@ func (m AlertMapper) IsSupported(version string) bool { // GetAlerts will make a request to Alertmanager API and parse the response // It will only return alerts or error (if any) -func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) { +func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) { groups := []models.AlertGroup{} receivers := map[string]alertsGroupReceiver{} resp := alertsGroupsAPISchema{} - url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups") + url, err := transport.JoinURL(uri, "api/v1/alerts/groups") if err != nil { return groups, err } - err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp) + err = transport.ReadJSON(url, timeout, &resp) if err != nil { return groups, err } diff --git a/mapper/v062/alerts.go b/mapper/v062/alerts.go index b20107330..7f2016b8e 100644 --- a/mapper/v062/alerts.go +++ b/mapper/v062/alerts.go @@ -10,7 +10,6 @@ import ( "time" "github.com/blang/semver" - "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/mapper" "github.com/cloudflare/unsee/models" "github.com/cloudflare/unsee/transport" @@ -65,17 +64,17 @@ func (m AlertMapper) IsSupported(version string) bool { // GetAlerts will make a request to Alertmanager API and parse the response // It will only return alerts or error (if any) -func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) { +func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) { groups := []models.AlertGroup{} receivers := map[string]alertsGroupReceiver{} resp := alertsGroupsAPISchema{} - url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups") + url, err := transport.JoinURL(uri, "api/v1/alerts/groups") if err != nil { return groups, err } - err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp) + err = transport.ReadJSON(url, timeout, &resp) if err != nil { return groups, err } diff --git a/models/models.go b/models/models.go index 467f5d92f..1677d4754 100644 --- a/models/models.go +++ b/models/models.go @@ -1,6 +1,11 @@ package models -import "time" +import ( + "crypto/sha1" + "fmt" + "io" + "time" +) // Silence is vanilla silence + some additional attributes // Unsee adds JIRA support, it can extract JIRA IDs from comments @@ -40,6 +45,15 @@ var AlertStateList = []string{ AlertStateSuppressed, } +// AlertmanagerUpstream describes the Alertmanager instance alert was collected +// from +type AlertmanagerUpstream struct { + Name string `json:"name"` + URI string `json:"uri"` + // all silences matching current alert in this upstream + Silences map[string]Silence `json:"silences"` +} + // Alert is vanilla alert + some additional attributes // unsee extends an alert object with: // * Links map, it's generated from annotations if annotation value is an url @@ -56,9 +70,11 @@ type Alert struct { SilencedBy []string `json:"silencedBy"` InhibitedBy []string `json:"inhibitedBy"` // unsee fields - Receiver string `json:"receiver"` - Links map[string]string `json:"links"` - Fingerprint string `json:"-"` + Alertmanager []AlertmanagerUpstream `json:"alertmanager"` + Receiver string `json:"receiver"` + Links map[string]string `json:"links"` + ID string `json:"-"` + Fingerprint string `json:"-"` } // IsSilenced will return true if alert should be considered silenced @@ -109,6 +125,15 @@ type AlertGroup struct { StateCount map[string]int `json:"stateCount"` } +// ContentFingerprint is a checksum of all alerts in the group +func (ag AlertGroup) ContentFingerprint() string { + h := sha1.New() + for _, alert := range ag.Alerts { + io.WriteString(h, alert.Fingerprint) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + // Filter holds returned data on any filter passed by the user as part of the query type Filter struct { Text string `json:"text"` @@ -139,15 +164,14 @@ type LabelsCountMap map[string]map[string]int // AlertsResponse is the structure of JSON response UI will use to get alert data type AlertsResponse struct { - Status string `json:"status"` - Error string `json:"error,omitempty"` - Timestamp string `json:"timestamp"` - Version string `json:"version"` - AlertGroups []AlertGroup `json:"groups"` - Silences map[string]Silence `json:"silences"` - Colors LabelsColorMap `json:"colors"` - Filters []Filter `json:"filters"` - Counters LabelsCountMap `json:"counters"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Timestamp string `json:"timestamp"` + Version string `json:"version"` + AlertGroups []AlertGroup `json:"groups"` + Colors LabelsColorMap `json:"colors"` + Filters []Filter `json:"filters"` + Counters LabelsCountMap `json:"counters"` } // Autocomplete is the structure of autocomplete object for filter hints diff --git a/store/store.go b/store/store.go deleted file mode 100644 index f13076f19..000000000 --- a/store/store.go +++ /dev/null @@ -1,44 +0,0 @@ -package store - -import ( - "sync" - - "github.com/cloudflare/unsee/models" -) - -type dataStore struct { - Lock sync.RWMutex - Groups []models.AlertGroup - Silences map[string]models.Silence - Colors models.LabelsColorMap - Autocomplete []models.Autocomplete -} - -// Store will keep all Alertmanager data we collect -var Store = dataStore{} - -// GetSilence returns silence data for specific silence id or nil if not found -func (ds *dataStore) GetSilence(s string) *models.Silence { - ds.Lock.RLock() - defer ds.Lock.RUnlock() - if silence, found := ds.Silences[s]; found { - return &silence - } - return nil -} - -// SetSilences allows to update silence list stored internally -func (ds *dataStore) SetSilences(s map[string]models.Silence) { - ds.Lock.Lock() - defer ds.Lock.Unlock() - ds.Silences = s -} - -// Update will lock the store and update internal data -func (ds *dataStore) Update(groups []models.AlertGroup, colors models.LabelsColorMap, autocomplete []models.Autocomplete) { - ds.Lock.Lock() - defer ds.Lock.Unlock() - ds.Groups = groups - ds.Colors = colors - ds.Autocomplete = autocomplete -} diff --git a/store/store_test.go b/store/store_test.go deleted file mode 100644 index f9a87c82d..000000000 --- a/store/store_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package store_test - -import ( - "testing" - - "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/store" -) - -type silenceTest struct { - silences map[string]models.Silence - silenceID string - found bool -} - -var silenceTests = []silenceTest{ - silenceTest{ - silences: map[string]models.Silence{ - "1": models.Silence{}, - }, - silenceID: "1", - found: true, - }, - silenceTest{ - silences: map[string]models.Silence{ - "1": models.Silence{}, - "2": models.Silence{}, - "3": models.Silence{}, - }, - silenceID: "2", - found: true, - }, - silenceTest{ - silences: map[string]models.Silence{}, - silenceID: "1", - found: false, - }, - silenceTest{ - silences: map[string]models.Silence{ - "2": models.Silence{}, - "3": models.Silence{}, - }, - silenceID: "1", - found: false, - }, -} - -func TestSilences(t *testing.T) { - for _, testCase := range silenceTests { - store.Store.SetSilences(testCase.silences) - silence := store.Store.GetSilence(testCase.silenceID) - found := silence != nil - if found != testCase.found { - t.Errorf("GetSilence('%s') returned %v, %v was expected", testCase.silenceID, found, testCase.found) - } - } -} diff --git a/timer.go b/timer.go index 2bc6c48b2..547266095 100644 --- a/timer.go +++ b/timer.go @@ -1,149 +1,37 @@ package main import ( - "crypto/sha1" - "fmt" - "io" "runtime" - "sort" + "sync" "github.com/cloudflare/unsee/alertmanager" - "github.com/cloudflare/unsee/config" - "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/store" - "github.com/cloudflare/unsee/transform" - "github.com/cnf/structhash" log "github.com/Sirupsen/logrus" - "github.com/prometheus/client_golang/prometheus" ) -// PullFromAlertmanager will try to fetch latest alerts and silences -// from Alertmanager API, it's called by Ticker timer -func PullFromAlertmanager() { +func pullFromAlertmanager() { // always flush cache once we're done defer apiCache.Flush() log.Info("Pulling latest alerts and silences from Alertmanager") - v := alertmanager.GetVersion() - silences, err := alertmanager.GetSilences(v) - if err != nil { - log.Error(err.Error()) - errorLock.Lock() - alertManagerError = err.Error() - errorLock.Unlock() - metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "silences"}).Inc() - return - } + upstreams := alertmanager.GetAlertmanagers() + wg := sync.WaitGroup{} + wg.Add(len(upstreams)) - alertGroups, err := alertmanager.GetAlerts(v) - if err != nil { - log.Error(err.Error()) - errorLock.Lock() - alertManagerError = err.Error() - errorLock.Unlock() - metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "alerts"}).Inc() - return - } - - log.Infof("Detecting JIRA links in silences (%d)", len(silences)) - silenceStore := make(map[string]models.Silence) - for _, silence := range silences { - silence.JiraID, silence.JiraURL = transform.DetectJIRAs(&silence) - silenceStore[silence.ID] = silence - } - - log.Infof("Updating list of stored silences (%d)", len(silenceStore)) - store.Store.SetSilences(silenceStore) - - alertStore := []models.AlertGroup{} - colorStore := make(models.LabelsColorMap) - - acMap := map[string]models.Autocomplete{} - - for _, state := range models.AlertStateList { - metricAlerts.With(prometheus.Labels{"state": state}).Set(0) - } - - log.Infof("Deduplicating alert groups (%d)", len(alertGroups)) - uniqueGroups := map[string]models.AlertGroup{} - uniqueAlerts := map[string]map[string]models.Alert{} - for _, ag := range alertGroups { - agIDHasher := sha1.New() - io.WriteString(agIDHasher, ag.Receiver) - io.WriteString(agIDHasher, fmt.Sprintf("%x", structhash.Sha1(ag.Labels, 1))) - agID := fmt.Sprintf("%x", agIDHasher.Sum(nil)) - if _, found := uniqueGroups[agID]; !found { - uniqueGroups[agID] = models.AlertGroup{ - Receiver: ag.Receiver, - Labels: ag.Labels, - ID: agID, + for _, upstream := range upstreams { + go func(am *alertmanager.Alertmanager) { + log.Infof("[%s] Collecting alerts and silences", am.Name) + err := am.Pull() + if err != nil { + log.Errorf("[%s] %s", am.Name, err) } - } - for _, alert := range ag.Alerts { - // generate alert fingerprint from a raw, unaltered alert object - aID := fmt.Sprintf("%x", structhash.Sha1(alert, 1)) - if _, found := uniqueAlerts[agID]; !found { - uniqueAlerts[agID] = map[string]models.Alert{} - } - if _, found := uniqueAlerts[agID][aID]; !found { - alert.Fingerprint = aID - uniqueAlerts[agID][aID] = alert - } - } + wg.Done() + }(upstream) } - log.Infof("Processing unique alert groups (%d)", len(uniqueGroups)) - for _, ag := range uniqueGroups { - // used to generate group content hash - agHasher := sha1.New() + wg.Wait() - alerts := models.AlertList{} - for _, alert := range uniqueAlerts[ag.ID] { - - alert.Annotations, alert.Links = transform.DetectLinks(alert.Annotations) - alert.Labels = transform.StripLables(config.Config.StripLabels, alert.Labels) - - io.WriteString(agHasher, alert.Fingerprint) // alert group hasher - - transform.ColorLabel(colorStore, "@receiver", alert.Receiver) - for k, v := range alert.Labels { - transform.ColorLabel(colorStore, k, v) - } - alerts = append(alerts, alert) - - // update internal metrics - metricAlerts.With(prometheus.Labels{"state": alert.State}).Inc() - } - - for _, hint := range transform.BuildAutocomplete(alerts) { - acMap[hint.Value] = hint - } - - sort.Sort(&alerts) - ag.Alerts = alerts - - // Hash is a checksum of all alerts, used to tell when any alert in the group changed - ag.Hash = fmt.Sprintf("%x", agHasher.Sum(nil)) - - alertStore = append(alertStore, ag) - } - - log.Infof("Merging autocomplete data (%d)", len(acMap)) - acStore := []models.Autocomplete{} - for _, hint := range acMap { - acStore = append(acStore, hint) - } - - errorLock.Lock() - alertManagerError = "" - errorLock.Unlock() - - metricAlertGroups.Set(float64(len(alertStore))) - - log.Infof("Updating list of stored alert groups (%d)", len(alertStore)) - store.Store.Update(alertStore, colorStore, acStore) log.Info("Pull completed") runtime.GC() } @@ -153,7 +41,7 @@ func Tick() { for { select { case <-ticker.C: - PullFromAlertmanager() + pullFromAlertmanager() } } } diff --git a/views.go b/views.go index a31864ef3..6cbebd17a 100644 --- a/views.go +++ b/views.go @@ -1,19 +1,15 @@ package main import ( - "crypto/sha1" "encoding/json" - "fmt" - "io" "net/http" "sort" "strings" "time" + "github.com/cloudflare/unsee/alertmanager" "github.com/cloudflare/unsee/config" "github.com/cloudflare/unsee/models" - "github.com/cloudflare/unsee/store" - "github.com/cloudflare/unsee/transport" log "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" @@ -51,10 +47,11 @@ func index(c *gin.Context) { defaultUsed = false } - silencesAPI, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences") - if err != nil { - log.Errorf("Can't generate silences API URL: %s", err) - } + // FIXME + //silencesAPI, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences") + //if err != nil { + // log.Errorf("Can't generate silences API URL: %s", err) + //} c.HTML(http.StatusOK, "templates/index.html", gin.H{ "Version": version, @@ -67,7 +64,7 @@ func index(c *gin.Context) { "DefaultUsed": defaultUsed, "StaticColorLabels": strings.Join(config.Config.ColorLabelsStatic, " "), "WebPrefix": config.Config.WebPrefix, - "SilencesApi": silencesAPI, + "SilencesApi": "FIXME", }) log.Infof("[%s] %s %s took %s", c.ClientIP(), c.Request.Method, c.Request.RequestURI, time.Since(start)) @@ -103,9 +100,7 @@ func alerts(c *gin.Context) { resp.Version = version // update error field, needs a lock - errorLock.RLock() - resp.Error = string(alertManagerError) - errorLock.RUnlock() + // FIXME resp.Error = string(alertManagerError) if resp.Error != "" { apiCache.Flush() @@ -127,13 +122,14 @@ func alerts(c *gin.Context) { // set pointers for data store objects, need a lock until end of view is reached alerts := []models.AlertGroup{} - silences := map[string]models.Silence{} colors := models.LabelsColorMap{} counters := models.LabelsCountMap{} - store.Store.Lock.RLock() + + dedupedAlerts := alertmanager.DedupAlerts() + dedupedColors := alertmanager.DedupColors() var matches int - for _, ag := range store.Store.Groups { + for _, ag := range dedupedAlerts { agCopy := models.AlertGroup{ ID: ag.ID, Receiver: ag.Receiver, @@ -144,7 +140,6 @@ func alerts(c *gin.Context) { for _, s := range models.AlertStateList { agCopy.StateCount[s] = 0 } - h := sha1.New() for _, alert := range ag.Alerts { results := []bool{} @@ -159,25 +154,11 @@ func alerts(c *gin.Context) { if !validFilters || (boolInSlice(results, true) && !boolInSlice(results, false)) { matches++ agCopy.Alerts = append(agCopy.Alerts, alert) - aj, err := json.Marshal(alert) - if err != nil { - log.Error(err.Error()) - panic(err.Error()) - } - io.WriteString(h, string(aj)) - - if alert.IsSilenced() { - for _, silenceID := range alert.SilencedBy { - if silence := store.Store.GetSilence(silenceID); silence != nil { - silences[silenceID] = *silence - } - } - } countLabel(counters, "@state", alert.State) countLabel(counters, "@receiver", alert.Receiver) - if ck, foundKey := store.Store.Colors["@receiver"]; foundKey { + if ck, foundKey := dedupedColors["@receiver"]; foundKey { if cv, foundVal := ck[alert.Receiver]; foundVal { if _, found := colors["@receiver"]; !found { colors["@receiver"] = map[string]models.LabelColors{} @@ -189,7 +170,7 @@ func alerts(c *gin.Context) { agCopy.StateCount[alert.State]++ for key, value := range alert.Labels { - if keyMap, foundKey := store.Store.Colors[key]; foundKey { + if keyMap, foundKey := dedupedColors[key]; foundKey { if color, foundColor := keyMap[value]; foundColor { if _, found := colors[key]; !found { colors[key] = map[string]models.LabelColors{} @@ -203,14 +184,13 @@ func alerts(c *gin.Context) { } if len(agCopy.Alerts) > 0 { - agCopy.Hash = fmt.Sprintf("%x", h.Sum(nil)) + agCopy.Hash = agCopy.ContentFingerprint() alerts = append(alerts, agCopy) } } resp.AlertGroups = alerts - resp.Silences = silences resp.Colors = colors resp.Counters = counters @@ -230,7 +210,6 @@ func alerts(c *gin.Context) { panic(err) } apiCache.Set(cacheKey, data, -1) - store.Store.Lock.RUnlock() c.Data(http.StatusOK, gin.MIMEJSON, data.([]byte)) logAlertsView(c, "MIS", time.Since(start)) @@ -264,8 +243,9 @@ func autocomplete(c *gin.Context) { acData := sort.StringSlice{} - store.Store.Lock.RLock() - for _, hint := range store.Store.Autocomplete { + dedupedAutocomplete := alertmanager.DedupAutocomplete() + + for _, hint := range dedupedAutocomplete { if strings.HasPrefix(strings.ToLower(hint.Value), strings.ToLower(term)) { acData = append(acData, hint.Value) } else { @@ -276,7 +256,6 @@ func autocomplete(c *gin.Context) { } } } - store.Store.Lock.RUnlock() sort.Sort(sort.Reverse(acData)) data, err := json.Marshal(acData) diff --git a/views_test.go b/views_test.go index 3b6c4d1b7..899ce90f1 100644 --- a/views_test.go +++ b/views_test.go @@ -31,9 +31,10 @@ func stringInSlice(stringArray []string, value string) bool { func mockConfig() { log.SetLevel(log.ErrorLevel) - os.Setenv("ALERTMANAGER_URI", "http://localhost") + os.Setenv("ALERTMANAGER_URI", "default:http://localhost") os.Setenv("COLOR_LABELS_UNIQUE", "alertname") config.Config.Read() + setupUpstreams() } func ginTestEngine() *gin.Engine { @@ -108,7 +109,7 @@ func mockAlerts(version string) { mock.RegisterURL("http://localhost/api/v1/silences", version, "silences") mock.RegisterURL("http://localhost/api/v1/alerts/groups", version, "alerts/groups") - PullFromAlertmanager() + pullFromAlertmanager() } func TestAlerts(t *testing.T) { @@ -126,22 +127,19 @@ func TestAlerts(t *testing.T) { ur := models.AlertsResponse{} json.Unmarshal(resp.Body.Bytes(), &ur) if len(ur.Filters) != 3 { - t.Errorf("[%s] No filters in response", version) + t.Errorf("[%s] Got %d filter(s) in response, expected %d", version, len(ur.Filters), 3) } if len(ur.Colors) != 1 { - t.Errorf("[%s] No colors in response", version) - } - if len(ur.Silences) != 1 { - t.Errorf("[%s] No silences in response", version) + t.Errorf("[%s] Got %d color(s) in response, expected %d", version, len(ur.Colors), 1) } if len(ur.AlertGroups) != 1 { - t.Errorf("[%s] No alerts in response", version) + t.Errorf("[%s] Got %d alert(s) in response, expected %d", version, len(ur.AlertGroups), 1) } if ur.Version == "" { - t.Errorf("[%s] No version in response", version) + t.Errorf("[%s] Empty version in response", version) } if ur.Timestamp == "" { - t.Errorf("[%s] No timestamp in response", version) + t.Errorf("[%s] Empty timestamp in response", version) } if ur.Error != "" { t.Errorf("[%s] Error in response: %s", version, ur.Error)