Refactor Alertmanager API client code to use multiple upstream instances

Alerts are stored per instance and deduplicated on read.

CONTRIBUTING.md

@@ -83,4 +83,4 @@ To support a new release that breaks API following changes needs to be done:
   silences (depending if both need a new code) under mapper/vXY (X major
   Alertmanager version, Y minor version).
 * Register new mapper in the `init()` function in the
-  `alertmanager/alertmanager.go` file.
+  `alertmanager/mapper.go` file.

Makefile

@@ -3,7 +3,7 @@ VERSION := $(shell git describe --tags --always --dirty='-dev')
 
 # Alertmanager instance used when running locally, points to mock data
 MOCK_PATH        := $(CURDIR)/mock/0.7.1
-ALERTMANAGER_URI := file://$(MOCK_PATH)
+ALERTMANAGER_URI := "localfs:file://$(MOCK_PATH) local2:file://$(CURDIR)/mock/0.7.0"
 # Listen port when running locally
 PORT := 8080
 

alertmanager/alertmanager.go

@@ -1,62 +0,0 @@
-package alertmanager
-
-import (
-	"time"
-
-	"github.com/cloudflare/unsee/mapper"
-	"github.com/cloudflare/unsee/mapper/v04"
-	"github.com/cloudflare/unsee/mapper/v05"
-	"github.com/cloudflare/unsee/mapper/v061"
-	"github.com/cloudflare/unsee/mapper/v062"
-	"github.com/cloudflare/unsee/models"
-
-	log "github.com/Sirupsen/logrus"
-)
-
-// initialize all mappers
-func init() {
-	mapper.RegisterAlertMapper(v04.AlertMapper{})
-	mapper.RegisterAlertMapper(v05.AlertMapper{})
-	mapper.RegisterAlertMapper(v061.AlertMapper{})
-	mapper.RegisterAlertMapper(v062.AlertMapper{})
-	mapper.RegisterSilenceMapper(v04.SilenceMapper{})
-	mapper.RegisterSilenceMapper(v05.SilenceMapper{})
-}
-
-// GetAlerts will send request to Alertmanager and return list of alert groups
-// from the API
-func GetAlerts(version string) ([]models.AlertGroup, error) {
-	groups := []models.AlertGroup{}
-
-	mapper, err := mapper.GetAlertMapper(version)
-	if err != nil {
-		return groups, err
-	}
-
-	start := time.Now()
-	groups, err = mapper.GetAlerts()
-	if err != nil {
-		return groups, err
-	}
-	log.Infof("Got %d alert group(s) in %s", len(groups), time.Since(start))
-	return groups, nil
-}
-
-// GetSilences will send request to Alertmanager and return list of silences
-// from the API
-func GetSilences(version string) ([]models.Silence, error) {
-	silences := []models.Silence{}
-
-	mapper, err := mapper.GetSilenceMapper(version)
-	if err != nil {
-		return silences, err
-	}
-
-	start := time.Now()
-	silences, err = mapper.GetSilences()
-	if err != nil {
-		return silences, err
-	}
-	log.Infof("Got %d silences(s) in %s", len(silences), time.Since(start))
-	return silences, nil
-}

alertmanager/alertmanager_test.go

@@ -1,66 +0,0 @@
-package alertmanager_test
-
-import (
-	"testing"
-
-	"github.com/cloudflare/unsee/alertmanager"
-	"github.com/cloudflare/unsee/config"
-	"github.com/cloudflare/unsee/mock"
-
-	log "github.com/Sirupsen/logrus"
-	httpmock "gopkg.in/jarcoal/httpmock.v1"
-)
-
-func TestGetAlerts(t *testing.T) {
-	log.SetLevel(log.ErrorLevel)
-	config.Config.AlertmanagerURI = "http://localhost"
-
-	httpmock.Activate()
-	defer httpmock.DeactivateAndReset()
-
-	for _, version := range mock.ListAllMocks() {
-		httpmock.Reset()
-		mock.RegisterURL("http://localhost/api/v1/status", version, "status")
-		mock.RegisterURL("http://localhost/api/v1/alerts/groups", version, "alerts/groups")
-
-		v := alertmanager.GetVersion()
-		if v != version {
-			t.Errorf("GetVersion() returned '%s', expected '%s'", v, version)
-		}
-
-		groups, err := alertmanager.GetAlerts(v)
-		if err != nil {
-			t.Errorf("GetAlerts(%s) failed: %s", version, err.Error())
-		}
-		if len(groups) != 10 {
-			t.Errorf("Got %d groups, expected 10", len(groups))
-		}
-	}
-}
-
-func TestGetSilences(t *testing.T) {
-	log.SetLevel(log.ErrorLevel)
-	config.Config.AlertmanagerURI = "http://localhost"
-
-	httpmock.Activate()
-	defer httpmock.DeactivateAndReset()
-
-	for _, version := range mock.ListAllMocks() {
-		httpmock.Reset()
-		mock.RegisterURL("http://localhost/api/v1/status", version, "status")
-		mock.RegisterURL("http://localhost/api/v1/silences", version, "silences")
-
-		v := alertmanager.GetVersion()
-		if v != version {
-			t.Errorf("GetVersion() returned '%s', expected '%s'", v, version)
-		}
-
-		silences, err := alertmanager.GetSilences(v)
-		if err != nil {
-			t.Errorf("GetSilences(%s) failed: %s", version, err.Error())
-		}
-		if len(silences) != 3 {
-			t.Errorf("Got %d silences, expected %d", len(silences), 3)
-		}
-	}
-}

alertmanager/dedup.go

@@ -0,0 +1,111 @@
+package alertmanager
+
+import (
+	"sort"
+
+	"github.com/cloudflare/unsee/models"
+)
+
+// DedupAlerts will collect alert groups from all defined Alertmanager
+// upstreams and deduplicate them, so we only return unique alerts
+func DedupAlerts() []models.AlertGroup {
+	uniqueGroups := map[string][]models.AlertGroup{}
+
+	upstreams := GetAlertmanagers()
+	for _, am := range upstreams {
+		groups := am.Alerts()
+		for _, ag := range groups {
+			if _, found := uniqueGroups[ag.ID]; !found {
+				uniqueGroups[ag.ID] = []models.AlertGroup{}
+			}
+			uniqueGroups[ag.ID] = append(uniqueGroups[ag.ID], ag)
+		}
+	}
+
+	dedupedGroups := []models.AlertGroup{}
+	for _, agList := range uniqueGroups {
+		alerts := map[string]models.Alert{}
+		for _, ag := range agList {
+			for _, alert := range ag.Alerts {
+				a, found := alerts[alert.ID]
+				if found {
+					for _, am := range alert.Alertmanager {
+						a.Alertmanager = append(a.Alertmanager, am)
+						alerts[alert.ID] = a
+					}
+				} else {
+					alerts[alert.ID] = alert
+				}
+			}
+		}
+		ag := models.AlertGroup(agList[0])
+		ag.Alerts = models.AlertList{}
+		for _, alert := range alerts {
+			ag.Alerts = append(ag.Alerts, alert)
+		}
+		sort.Sort(ag.Alerts)
+		ag.Hash = ag.ContentFingerprint()
+		dedupedGroups = append(dedupedGroups, ag)
+	}
+
+	return dedupedGroups
+}
+
+// DedupColors returns a color map merged from all Alertmanager upstream color
+// maps
+func DedupColors() models.LabelsColorMap {
+	dedupedColors := models.LabelsColorMap{}
+
+	upstreams := GetAlertmanagers()
+
+	for _, am := range upstreams {
+		colors := am.Colors()
+		// map[string]map[string]LabelColors
+		for labelName, valueMap := range colors {
+			if _, found := dedupedColors[labelName]; !found {
+				dedupedColors[labelName] = map[string]models.LabelColors{}
+			}
+			for labelVal, labelColors := range valueMap {
+				if _, found := dedupedColors[labelName][labelVal]; !found {
+					dedupedColors[labelName][labelVal] = labelColors
+				}
+			}
+		}
+	}
+
+	return dedupedColors
+}
+
+// DedupAutocomplete returns a list of autocomplete hints merged from all
+// Alertmanager upstreams
+func DedupAutocomplete() []models.Autocomplete {
+	dedupedAutocomplete := []models.Autocomplete{}
+	uniqueAutocomplete := map[string]*models.Autocomplete{}
+
+	upstreams := GetAlertmanagers()
+
+	for _, am := range upstreams {
+		ac := am.Autocomplete()
+		for _, hint := range ac {
+			h, found := uniqueAutocomplete[hint.Value]
+			if found {
+				for _, token := range hint.Tokens {
+					if !stringInSlice(h.Tokens, token) {
+						h.Tokens = append(h.Tokens, token)
+					}
+				}
+			} else {
+				uniqueAutocomplete[hint.Value] = &models.Autocomplete{
+					Value:  hint.Value,
+					Tokens: hint.Tokens,
+				}
+			}
+		}
+	}
+
+	for _, hint := range uniqueAutocomplete {
+		dedupedAutocomplete = append(dedupedAutocomplete, *hint)
+	}
+
+	return dedupedAutocomplete
+}

alertmanager/mapper.go

@@ -0,0 +1,19 @@
+package alertmanager
+
+import (
+	"github.com/cloudflare/unsee/mapper"
+	"github.com/cloudflare/unsee/mapper/v04"
+	"github.com/cloudflare/unsee/mapper/v05"
+	"github.com/cloudflare/unsee/mapper/v061"
+	"github.com/cloudflare/unsee/mapper/v062"
+)
+
+// initialize all mappers
+func init() {
+	mapper.RegisterAlertMapper(v04.AlertMapper{})
+	mapper.RegisterAlertMapper(v05.AlertMapper{})
+	mapper.RegisterAlertMapper(v061.AlertMapper{})
+	mapper.RegisterAlertMapper(v062.AlertMapper{})
+	mapper.RegisterSilenceMapper(v04.SilenceMapper{})
+	mapper.RegisterSilenceMapper(v05.SilenceMapper{})
+}

alertmanager/models.go

@@ -0,0 +1,281 @@
+package alertmanager
+
+import (
+	"crypto/sha1"
+	"fmt"
+	"io"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/unsee/config"
+	"github.com/cloudflare/unsee/mapper"
+	"github.com/cloudflare/unsee/models"
+	"github.com/cloudflare/unsee/transform"
+	"github.com/cloudflare/unsee/transport"
+	"github.com/cnf/structhash"
+	"github.com/prometheus/client_golang/prometheus"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+// Alertmanager represents Alertmanager upstream instance
+type Alertmanager struct {
+	URI     string        `json:"uri"`
+	Timeout time.Duration `json:"timeout"`
+	Name    string        `json:"name"`
+	// lock protects data access while updating
+	lock sync.RWMutex
+	// fields for storing pulled data
+	alertGroups  []models.AlertGroup
+	silences     map[string]models.Silence
+	colors       models.LabelsColorMap
+	autocomplete []models.Autocomplete
+}
+
+func (am *Alertmanager) detectVersion() string {
+	// if everything fails assume Alertmanager is at latest possible version
+	defaultVersion := "999.0.0"
+
+	url, err := transport.JoinURL(am.URI, "api/v1/status")
+	if err != nil {
+		log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", am.URI, err)
+		return defaultVersion
+	}
+	ver := alertmanagerVersion{}
+	err = transport.ReadJSON(url, am.Timeout, &ver)
+	if err != nil {
+		log.Errorf("[%s] %s request failed: %s", am.Name, url, err.Error())
+		return defaultVersion
+	}
+
+	if ver.Status != "success" {
+		log.Errorf("[%s] Request to %s returned status %s", am.Name, url, ver.Status)
+		return defaultVersion
+	}
+
+	if ver.Data.VersionInfo.Version == "" {
+		log.Errorf("[%s] No version information in Alertmanager API at %s", am.Name, url)
+		return defaultVersion
+	}
+
+	log.Infof("[%s] Remote Alertmanager version: %s", am.Name, ver.Data.VersionInfo.Version)
+	return ver.Data.VersionInfo.Version
+}
+
+func (am *Alertmanager) pullSilences(version string) error {
+	mapper, err := mapper.GetSilenceMapper(version)
+	if err != nil {
+		return err
+	}
+
+	start := time.Now()
+	silences, err := mapper.GetSilences(am.URI, am.Timeout)
+	if err != nil {
+		return err
+	}
+	log.Infof("[%s] Got %d silences(s) in %s", am.Name, len(silences), time.Since(start))
+
+	log.Infof("[%s] Detecting JIRA links in silences (%d)", am.Name, len(silences))
+	silenceMap := map[string]models.Silence{}
+	for _, silence := range silences {
+		silence.JiraID, silence.JiraURL = transform.DetectJIRAs(&silence)
+		silenceMap[silence.ID] = silence
+	}
+
+	am.lock.Lock()
+	am.silences = silenceMap
+	am.lock.Unlock()
+
+	return nil
+}
+
+func (am *Alertmanager) pullAlerts(version string) error {
+	mapper, err := mapper.GetAlertMapper(version)
+	if err != nil {
+		return err
+	}
+
+	start := time.Now()
+	groups, err := mapper.GetAlerts(am.URI, am.Timeout)
+	if err != nil {
+		return err
+	}
+	log.Infof("[%s] Got %d alert group(s) in %s", am.Name, len(groups), time.Since(start))
+
+	log.Infof("[%s] Deduplicating alert groups (%d)", am.Name, len(groups))
+	uniqueGroups := map[string]models.AlertGroup{}
+	uniqueAlerts := map[string]map[string]models.Alert{}
+	for _, ag := range groups {
+		agIDHasher := sha1.New()
+		io.WriteString(agIDHasher, ag.Receiver)
+		io.WriteString(agIDHasher, fmt.Sprintf("%x", structhash.Sha1(ag.Labels, 1)))
+		agID := fmt.Sprintf("%x", agIDHasher.Sum(nil))
+		if _, found := uniqueGroups[agID]; !found {
+			uniqueGroups[agID] = models.AlertGroup{
+				Receiver: ag.Receiver,
+				Labels:   ag.Labels,
+				ID:       agID,
+			}
+		}
+		for _, alert := range ag.Alerts {
+			// generate alert id from labels
+			aID := fmt.Sprintf("%x", structhash.Sha1(alert.Labels, 1))
+			alert.ID = aID
+			// generate alert fingerprint from a raw, unaltered alert object
+			fp := fmt.Sprintf("%x", structhash.Sha1(alert, 1))
+			if _, found := uniqueAlerts[agID]; !found {
+				uniqueAlerts[agID] = map[string]models.Alert{}
+			}
+			if _, found := uniqueAlerts[agID][fp]; !found {
+				alert.Fingerprint = fp
+				uniqueAlerts[agID][fp] = alert
+			}
+		}
+
+	}
+
+	dedupedGroups := []models.AlertGroup{}
+	colors := models.LabelsColorMap{}
+	autocompleteMap := map[string]models.Autocomplete{}
+	log.Infof("[%s] Processing unique alert groups (%d)", am.Name, len(uniqueGroups))
+	for _, ag := range uniqueGroups {
+		alerts := models.AlertList{}
+		for _, alert := range uniqueAlerts[ag.ID] {
+
+			silences := map[string]models.Silence{}
+			for _, silenceID := range alert.SilencedBy {
+				silence, err := am.SilenceByID(silenceID)
+				if err == nil {
+					silences[silenceID] = silence
+				}
+			}
+			alert.Alertmanager = []models.AlertmanagerUpstream{
+				models.AlertmanagerUpstream{
+					Name:     am.Name,
+					URI:      am.URI,
+					Silences: silences,
+				},
+			}
+
+			alert.Annotations, alert.Links = transform.DetectLinks(alert.Annotations)
+			alert.Labels = transform.StripLables(config.Config.StripLabels, alert.Labels)
+
+			transform.ColorLabel(colors, "@receiver", alert.Receiver)
+			for k, v := range alert.Labels {
+				transform.ColorLabel(colors, k, v)
+			}
+			alerts = append(alerts, alert)
+
+			// update internal metrics
+			metricAlerts.With(prometheus.Labels{
+				"alertmanager": am.Name,
+				"state":        alert.State,
+			}).Inc()
+		}
+
+		for _, hint := range transform.BuildAutocomplete(alerts) {
+			autocompleteMap[hint.Value] = hint
+		}
+
+		sort.Sort(&alerts)
+		ag.Alerts = alerts
+
+		// Hash is a checksum of all alerts, used to tell when any alert in the group changed
+		ag.Hash = ag.ContentFingerprint()
+
+		dedupedGroups = append(dedupedGroups, ag)
+	}
+
+	log.Infof("[%s] Merging autocomplete data (%d)", am.Name, len(autocompleteMap))
+	autocomplete := []models.Autocomplete{}
+	for _, hint := range autocompleteMap {
+		autocomplete = append(autocomplete, hint)
+	}
+
+	am.lock.Lock()
+	am.alertGroups = dedupedGroups
+	am.colors = colors
+	am.autocomplete = autocomplete
+	am.lock.Unlock()
+
+	metricAlertGroups.With(prometheus.Labels{
+		"alertmanager": am.Name,
+	}).Set(float64(len(dedupedGroups)))
+
+	return nil
+}
+
+// Pull data from upstream Alertmanager instance
+func (am *Alertmanager) Pull() error {
+	version := am.detectVersion()
+
+	err := am.pullSilences(version)
+	if err != nil {
+		metricAlertmanagerErrors.With(prometheus.Labels{
+			"alertmanager": am.Name,
+			"endpoint":     "silences",
+		}).Inc()
+		return err
+	}
+
+	err = am.pullAlerts(version)
+	if err != nil {
+		metricAlertmanagerErrors.With(prometheus.Labels{
+			"alertmanager": am.Name,
+			"endpoint":     "alerts",
+		}).Inc()
+		return err
+	}
+
+	return nil
+}
+
+// Alerts returns a copy of all alert groups
+func (am *Alertmanager) Alerts() []models.AlertGroup {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	alerts := make([]models.AlertGroup, len(am.alertGroups))
+	copy(alerts, am.alertGroups)
+	return alerts
+}
+
+// SilenceByID allows to query for a silence by it's ID, returns error if not found
+func (am *Alertmanager) SilenceByID(id string) (models.Silence, error) {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	for k, v := range am.silences {
+		if k == id {
+			return models.Silence(v), nil
+		}
+	}
+
+	return models.Silence{}, fmt.Errorf("Silence '%s' not found", id)
+}
+
+// Colors returns a copy of all color maps
+func (am *Alertmanager) Colors() models.LabelsColorMap {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	colors := models.LabelsColorMap{}
+	for k, v := range am.colors {
+		colors[k] = map[string]models.LabelColors{}
+		for nk, nv := range v {
+			colors[k][nk] = nv
+		}
+	}
+	return colors
+}
+
+// Autocomplete returns a copy of all autocomplete data
+func (am *Alertmanager) Autocomplete() []models.Autocomplete {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	autocomplete := make([]models.Autocomplete, len(am.autocomplete))
+	copy(autocomplete, am.autocomplete)
+	return autocomplete
+}

alertmanager/slices.go

@@ -0,0 +1,19 @@
+package alertmanager
+
+func boolInSlice(boolArray []bool, value bool) bool {
+	for _, s := range boolArray {
+		if s == value {
+			return true
+		}
+	}
+	return false
+}
+
+func stringInSlice(stringArray []string, value string) bool {
+	for _, s := range stringArray {
+		if s == value {
+			return true
+		}
+	}
+	return false
+}

alertmanager/upstream.go

@@ -1,40 +1,26 @@
 package alertmanager
 
 import (
-	"crypto/sha1"
 	"fmt"
-	"io"
-	"sort"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/unsee/config"
-	"github.com/cloudflare/unsee/mapper"
 	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/transform"
-	"github.com/cloudflare/unsee/transport"
-	"github.com/cnf/structhash"
 	"github.com/prometheus/client_golang/prometheus"
 
 	log "github.com/Sirupsen/logrus"
 )
 
-// Alertmanager represents Alertmanager upstream instance
-type Alertmanager struct {
-	URI     string        `json:"uri"`
-	Timeout time.Duration `json:"timeout"`
-	Name    string        `json:"name"`
-	// lock protects data access while updating
-	lock sync.RWMutex
-	// fields for storing pulled data
-	alertGroups  []models.AlertGroup
-	silences     map[string]models.Silence
-	colors       models.LabelsColorMap
-	autocomplete []models.Autocomplete
-}
+var (
+	upstreams = map[string]*Alertmanager{}
+)
 
 // NewAlertmanager creates a new Alertmanager instance
-func NewAlertmanager(name, uri string, timeout time.Duration) Alertmanager {
+func NewAlertmanager(name, uri string, timeout time.Duration) error {
+	if _, found := upstreams[name]; found {
+		return fmt.Errorf("Alertmanager upstream '%s' already exist", name)
+	}
+
 	// initialize metrics
 	metricAlertmanagerErrors.With(prometheus.Labels{
 		"alertmanager": name,
@@ -45,7 +31,7 @@ func NewAlertmanager(name, uri string, timeout time.Duration) Alertmanager {
 		"endpoint":     "silences",
 	}).Set(0)
 
-	return Alertmanager{
+	upstreams[name] = &Alertmanager{
 		URI:          uri,
 		Timeout:      timeout,
 		Name:         name,
@@ -55,217 +41,27 @@ func NewAlertmanager(name, uri string, timeout time.Duration) Alertmanager {
 		colors:       models.LabelsColorMap{},
 		autocomplete: []models.Autocomplete{},
 	}
-}
 
-func (am *Alertmanager) detectVersion() string {
-	// if everything fails assume Alertmanager is at latest possible version
-	defaultVersion := "999.0.0"
-
-	url, err := transport.JoinURL(am.URI, "api/v1/status")
-	if err != nil {
-		log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", am.URI, err)
-		return defaultVersion
-	}
-	ver := alertmanagerVersion{}
-	err = transport.ReadJSON(url, am.Timeout, &ver)
-	if err != nil {
-		log.Errorf("[%s] %s request failed: %s", am.Name, url, err.Error())
-		return defaultVersion
-	}
-
-	if ver.Status != "success" {
-		log.Errorf("[%s] Request to %s returned status %s", am.Name, url, ver.Status)
-		return defaultVersion
-	}
-
-	if ver.Data.VersionInfo.Version == "" {
-		log.Errorf("[%s] No version information in Alertmanager API at %s", am.Name, url)
-		return defaultVersion
-	}
-
-	log.Infof("[%s] Remote Alertmanager version: %s", am.Name, ver.Data.VersionInfo.Version)
-	return ver.Data.VersionInfo.Version
-}
-
-func (am *Alertmanager) pullSilences(version string) error {
-	mapper, err := mapper.GetSilenceMapper(version)
-	if err != nil {
-		return err
-	}
-
-	start := time.Now()
-	silences, err := mapper.GetSilences()
-	if err != nil {
-		return err
-	}
-	log.Infof("[%s] Got %d silences(s) in %s", am.Name, len(silences), time.Since(start))
-
-	log.Infof("[%s] Detecting JIRA links in silences (%d)", am.Name, len(silences))
-	silenceMap := map[string]models.Silence{}
-	for _, silence := range silences {
-		silence.JiraID, silence.JiraURL = transform.DetectJIRAs(&silence)
-		silenceMap[silence.ID] = silence
-	}
-
-	am.lock.Lock()
-	am.silences = silenceMap
-	am.lock.Unlock()
+	log.Infof("[%s] Configured Alertmanager source at %s", name, uri)
 
 	return nil
 }
 
-func (am *Alertmanager) pullAlerts(version string) error {
-	mapper, err := mapper.GetAlertMapper(version)
-	if err != nil {
-		return err
+// GetAlertmanagers returns a list of all defined Alertmanager instances
+func GetAlertmanagers() []*Alertmanager {
+	ams := []*Alertmanager{}
+	for _, am := range upstreams {
+		ams = append(ams, am)
 	}
+	return ams
+}
 
-	start := time.Now()
-	groups, err := mapper.GetAlerts()
-	if err != nil {
-		return err
+// GetAlertmanagerByName returns an instance of Alertmanager by name or nil
+// if not found
+func GetAlertmanagerByName(name string) *Alertmanager {
+	am, found := upstreams[name]
+	if found {
+		return am
 	}
-	log.Infof("[%s] Got %d alert group(s) in %s", am.Name, len(groups), time.Since(start))
-
-	log.Infof("[%s] Deduplicating alert groups (%d)", am.Name, len(groups))
-	uniqueGroups := map[string]models.AlertGroup{}
-	uniqueAlerts := map[string]map[string]models.Alert{}
-	for _, ag := range groups {
-		agIDHasher := sha1.New()
-		io.WriteString(agIDHasher, ag.Receiver)
-		io.WriteString(agIDHasher, fmt.Sprintf("%x", structhash.Sha1(ag.Labels, 1)))
-		agID := fmt.Sprintf("%x", agIDHasher.Sum(nil))
-		if _, found := uniqueGroups[agID]; !found {
-			uniqueGroups[agID] = models.AlertGroup{
-				Receiver: ag.Receiver,
-				Labels:   ag.Labels,
-				ID:       agID,
-			}
-		}
-		for _, alert := range ag.Alerts {
-			// generate alert fingerprint from a raw, unaltered alert object
-			aID := fmt.Sprintf("%x", structhash.Sha1(alert, 1))
-			if _, found := uniqueAlerts[agID]; !found {
-				uniqueAlerts[agID] = map[string]models.Alert{}
-			}
-			if _, found := uniqueAlerts[agID][aID]; !found {
-				alert.Fingerprint = aID
-				uniqueAlerts[agID][aID] = alert
-			}
-		}
-
-	}
-
-	dedupedGroups := []models.AlertGroup{}
-	colors := models.LabelsColorMap{}
-	autocompleteMap := map[string]models.Autocomplete{}
-	log.Infof("[%s] Processing unique alert groups (%d)", am.Name, len(uniqueGroups))
-	for _, ag := range uniqueGroups {
-		// used to generate group content hash
-		agHasher := sha1.New()
-
-		alerts := models.AlertList{}
-		for _, alert := range uniqueAlerts[ag.ID] {
-
-			alert.Annotations, alert.Links = transform.DetectLinks(alert.Annotations)
-			alert.Labels = transform.StripLables(config.Config.StripLabels, alert.Labels)
-
-			io.WriteString(agHasher, alert.Fingerprint) // alert group hasher
-
-			transform.ColorLabel(colors, "@receiver", alert.Receiver)
-			for k, v := range alert.Labels {
-				transform.ColorLabel(colors, k, v)
-			}
-			alerts = append(alerts, alert)
-
-			// update internal metrics
-			metricAlerts.With(prometheus.Labels{
-				"alertmanager": am.Name,
-				"state":        alert.State,
-			}).Inc()
-		}
-
-		for _, hint := range transform.BuildAutocomplete(alerts) {
-			autocompleteMap[hint.Value] = hint
-		}
-
-		sort.Sort(&alerts)
-		ag.Alerts = alerts
-
-		// Hash is a checksum of all alerts, used to tell when any alert in the group changed
-		ag.Hash = fmt.Sprintf("%x", agHasher.Sum(nil))
-
-		dedupedGroups = append(dedupedGroups, ag)
-	}
-
-	log.Infof("[%s] Merging autocomplete data (%d)", am.Name, len(autocompleteMap))
-	autocomplete := []models.Autocomplete{}
-	for _, hint := range autocompleteMap {
-		autocomplete = append(autocomplete, hint)
-	}
-
-	am.lock.Lock()
-	am.alertGroups = dedupedGroups
-	am.colors = colors
-	am.autocomplete = autocomplete
-	am.lock.Unlock()
-
-	metricAlertGroups.With(prometheus.Labels{
-		"alertmanager": am.Name,
-	}).Set(float64(len(dedupedGroups)))
-
 	return nil
 }
-
-// Pull data from upstream Alertmanager instance
-func (am *Alertmanager) Pull() error {
-	version := am.detectVersion()
-
-	err := am.pullSilences(version)
-	if err != nil {
-		metricAlertmanagerErrors.With(prometheus.Labels{
-			"alertmanager": am.Name,
-			"endpoint":     "silences",
-		}).Inc()
-		return err
-	}
-
-	err = am.pullAlerts(version)
-	if err != nil {
-		metricAlertmanagerErrors.With(prometheus.Labels{
-			"alertmanager": am.Name,
-			"endpoint":     "alerts",
-		}).Inc()
-		return err
-	}
-
-	return nil
-}
-
-// Alerts returns a copy of all stored alert groups
-func (am *Alertmanager) Alerts() []models.AlertGroup {
-	am.lock.RLock()
-	defer am.lock.RUnlock()
-
-	alerts := make([]models.AlertGroup, len(am.alertGroups))
-	copy(alerts, am.alertGroups)
-	return alerts
-}
-
-// SilencesByID returns a map copy of id->silences for given list of silence IDs
-func (am *Alertmanager) SilencesByID(ids []string) map[string]models.Silence {
-	am.lock.RLock()
-	defer am.lock.RUnlock()
-
-	silences := map[string]models.Silence{}
-	for k, v := range am.silences {
-		for _, s := range ids {
-			if k == s {
-				silences[k] = v
-				break
-			}
-		}
-	}
-
-	return silences
-}

alertmanager/version.go

@@ -1,6 +1,8 @@
 package alertmanager
 
 import (
+	"time"
+
 	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/transport"
 
@@ -19,17 +21,17 @@ type alertmanagerVersion struct {
 }
 
 // GetVersion returns version information of the remote Alertmanager endpoint
-func GetVersion() string {
+func GetVersion(uri string, timeout time.Duration) string {
 	// if everything fails assume Alertmanager is at latest possible version
 	defaultVersion := "999.0.0"
 
-	url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/status")
+	url, err := transport.JoinURL(uri, "api/v1/status")
 	if err != nil {
 		log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", config.Config.AlertmanagerURI, err.Error())
 		return defaultVersion
 	}
 	ver := alertmanagerVersion{}
-	err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &ver)
+	err = transport.ReadJSON(url, timeout, &ver)
 	if err != nil {
 		log.Errorf("%s request failed: %s", url, err.Error())
 		return defaultVersion

assets/static/alerts.js

@@ -6,8 +6,7 @@
 /* exported Alerts */
 var Alerts = (function() {
 
-    var silences = {},
-        labelCache = new LRUMap(1000);
+    var labelCache = new LRUMap(1000);
 
     function AlertGroup(groupData) {
         $.extend(this, groupData);
@@ -16,7 +15,6 @@ var Alerts = (function() {
     AlertGroup.prototype.Render = function() {
         return Templates.Render("alertGroup", {
             group: this,
-            silences: silences,
             alertLimit: 5
         });
     };
@@ -105,9 +103,6 @@ var Alerts = (function() {
         var alertCount = 0;
         var groups = {};
 
-        // update global silences dict as it's needed for rendering
-        silences = apiResponse.silences;
-
         var summaryData = {};
         $.each(apiResponse.counters, function(labelKey, counters){
             $.each(counters, function(labelVal, hits){

assets/templates/alertgroup.html

@@ -110,10 +110,9 @@
   <small class="silence-comment-title text-muted">
     Silenced by:
   </small>
-  <% _.each(alert.silencedBy, function(silenceID) { %>
-    <div class="silence-block">
-      <% var silence = silences[silenceID] %>
-      <% if (silence) { %>
+  <% _.each(alert.alertmanager, function(am) { %>
+    <% _.each(am.silences, function(silence) { %>
+      <div class="silence-block">
         <blockquote class="silence-comment">
           <% if (silence.jiraURL) { %>
             <a href="<%= silence.jiraURL %>" target="_blank">
@@ -124,6 +123,9 @@
             <%- silence.comment %>
           <% } %>
           <br/>
+          <div class="label label-list label-primary">
+            <%- am.name %>
+          </div>
           <div class="label label-list label-default label-age label-ts cursor-help"
              data-toggle="tooltip"
              data-placement="top"
@@ -159,8 +161,8 @@
             </cite>
           </footer>
         </blockquote>
-      <% } %>
-    </div>
+      </div>
+    <% }) %>
   <% }) %>
 </script>
 
@@ -261,7 +263,7 @@
               <%= Templates.Render('alertGroupLabels', {alert: alert, group: group}) %>
               <%= Templates.Render('alertGroupElements', {alert: alert}) %>
               <% if (alert.silencedBy.length) { %>
-                <%= Templates.Render('alertGroupSilence', {alert: alert, silences: silences}) %>
+                <%= Templates.Render('alertGroupSilence', {alert: alert}) %>
               <% } %>
             </div>
           </div>

config/config.go

@@ -25,7 +25,7 @@ func (mvd *spaceSeparatedList) Decode(value string) error {
 type configEnvs struct {
 	AlertmanagerTimeout time.Duration      `envconfig:"ALERTMANAGER_TIMEOUT" default:"40s" help:"Timeout for all request send to Alertmanager"`
 	AlertmanagerTTL     time.Duration      `envconfig:"ALERTMANAGER_TTL" default:"1m" help:"TTL for Alertmanager alerts and silences"`
-	AlertmanagerURI     string             `envconfig:"ALERTMANAGER_URI" required:"true" help:"Alertmanager URI"`
+	AlertmanagerURI     spaceSeparatedList `envconfig:"ALERTMANAGER_URI" required:"true" help:"Alertmanager URIs"`
 	ColorLabelsStatic   spaceSeparatedList `envconfig:"COLOR_LABELS_STATIC" help:"List of label names that should have the same (but distinct) color"`
 	ColorLabelsUnique   spaceSeparatedList `envconfig:"COLOR_LABELS_UNIQUE" help:"List of label names that should have unique color"`
 	Debug               bool               `envconfig:"DEBUG" default:"false" help:"Enable debug mode"`

filters/filter_fuzzy.go

@@ -5,7 +5,6 @@ import (
 	"regexp"
 
 	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/store"
 )
 
 type fuzzyFilter struct {
@@ -42,10 +41,14 @@ func (filter *fuzzyFilter) Match(alert *models.Alert, matches int) bool {
 		}
 
 		for _, silenceID := range alert.SilencedBy {
-			if silence := store.Store.GetSilence(silenceID); silence != nil {
-				if filter.Matcher.Compare(silence.Comment, filter.Value) {
-					filter.Hits++
-					return true
+			for _, am := range alert.Alertmanager {
+				silence, found := am.Silences[silenceID]
+				if found {
+					m := filter.Matcher.Compare(silence.Comment, filter.Value)
+					if m {
+						filter.Hits++
+						return true
+					}
 				}
 			}
 		}

filters/filter_silence_author.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/store"
 )
 
 type silenceAuthorFilter struct {
@@ -17,8 +16,14 @@ func (filter *silenceAuthorFilter) Match(alert *models.Alert, matches int) bool
 		var isMatch bool
 		if alert.IsSilenced() {
 			for _, silenceID := range alert.SilencedBy {
-				if silence := store.Store.GetSilence(silenceID); silence != nil {
-					isMatch = filter.Matcher.Compare(filter.Value, silence.CreatedBy)
+				for _, am := range alert.Alertmanager {
+					silence, found := am.Silences[silenceID]
+					if found {
+						m := filter.Matcher.Compare(silence.CreatedBy, filter.Value)
+						if m {
+							isMatch = m
+						}
+					}
 				}
 			}
 		} else {
@@ -43,15 +48,18 @@ func sinceAuthorAutocomplete(name string, operators []string, alerts []models.Al
 	for _, alert := range alerts {
 		if alert.IsSilenced() {
 			for _, silenceID := range alert.SilencedBy {
-				if silence := store.Store.GetSilence(silenceID); silence != nil {
-					for _, operator := range operators {
-						token := fmt.Sprintf("%s%s%s", name, operator, silence.CreatedBy)
-						tokens[token] = makeAC(token, []string{
-							name,
-							strings.TrimPrefix(name, "@"),
-							fmt.Sprintf("%s%s", name, operator),
-							silence.CreatedBy,
-						})
+				for _, am := range alert.Alertmanager {
+					silence, found := am.Silences[silenceID]
+					if found {
+						for _, operator := range operators {
+							token := fmt.Sprintf("%s%s%s", name, operator, silence.CreatedBy)
+							tokens[token] = makeAC(token, []string{
+								name,
+								strings.TrimPrefix(name, "@"),
+								fmt.Sprintf("%s%s", name, operator),
+								silence.CreatedBy,
+							})
+						}
 					}
 				}
 			}

filters/filter_silence_jira.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/store"
 )
 
 type silenceJiraFilter struct {
@@ -17,8 +16,14 @@ func (filter *silenceJiraFilter) Match(alert *models.Alert, matches int) bool {
 		var isMatch bool
 		if alert.IsSilenced() {
 			for _, silenceID := range alert.SilencedBy {
-				if silence := store.Store.GetSilence(silenceID); silence != nil {
-					isMatch = filter.Matcher.Compare(silence.JiraID, filter.Value)
+				for _, am := range alert.Alertmanager {
+					silence, found := am.Silences[silenceID]
+					if found {
+						m := filter.Matcher.Compare(silence.JiraID, filter.Value)
+						if m {
+							isMatch = m
+						}
+					}
 				}
 			}
 		} else {
@@ -43,16 +48,18 @@ func sinceJiraIDAutocomplete(name string, operators []string, alerts []models.Al
 	for _, alert := range alerts {
 		if alert.IsSilenced() {
 			for _, silenceID := range alert.SilencedBy {
-				silence := store.Store.GetSilence(silenceID)
-				if silence != nil && silence.JiraID != "" {
-					for _, operator := range operators {
-						token := fmt.Sprintf("%s%s%s", name, operator, silence.JiraID)
-						tokens[token] = makeAC(token, []string{
-							name,
-							strings.TrimPrefix(name, "@"),
-							fmt.Sprintf("%s%s", name, operator),
-							silence.JiraID,
-						})
+				for _, am := range alert.Alertmanager {
+					silence, found := am.Silences[silenceID]
+					if found && silence.JiraID != "" {
+						for _, operator := range operators {
+							token := fmt.Sprintf("%s%s%s", name, operator, silence.JiraID)
+							tokens[token] = makeAC(token, []string{
+								name,
+								strings.TrimPrefix(name, "@"),
+								fmt.Sprintf("%s%s", name, operator),
+								silence.JiraID,
+							})
+						}
 					}
 				}
 			}

filters/filter_test.go

@@ -5,9 +5,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cloudflare/unsee/alertmanager"
 	"github.com/cloudflare/unsee/filters"
 	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/store"
+
+	log "github.com/Sirupsen/logrus"
 )
 
 type filterTest struct {
@@ -433,14 +435,27 @@ var tests = []filterTest{
 }
 
 func TestFilters(t *testing.T) {
+	log.SetLevel(log.ErrorLevel)
+
+	err := alertmanager.NewAlertmanager("test", "http://localhost", time.Second)
+	am := alertmanager.GetAlertmanagerByName("test")
+	if err != nil {
+		t.Error(err)
+	}
 	for _, ft := range tests {
+		alert := models.Alert(ft.Alert)
 		if &ft.Silence != nil {
-			store.Store.SetSilences(map[string]models.Silence{
-				ft.Silence.ID: ft.Silence,
-			})
-		} else {
-			store.Store.SetSilences(map[string]models.Silence{})
+			alert.Alertmanager = []models.AlertmanagerUpstream{
+				models.AlertmanagerUpstream{
+					Name: am.Name,
+					URI:  am.URI,
+					Silences: map[string]models.Silence{
+						ft.Silence.ID: ft.Silence,
+					},
+				},
+			}
 		}
+
 		f := filters.NewFilter(ft.Expression)
 		if f == nil {
 			t.Errorf("[%s] No filter found", ft.Expression)
@@ -452,7 +467,7 @@ func TestFilters(t *testing.T) {
 			t.Errorf("[%s] GetIsValid() returned %#v while %#v was expected", ft.Expression, f.GetIsValid(), ft.IsValid)
 		}
 		if f.GetIsValid() {
-			m := f.Match(&ft.Alert, 0)
+			m := f.Match(&alert, 0)
 			if m != ft.IsMatch {
 				j, _ := json.Marshal(ft.Alert)
 				s, _ := json.Marshal(ft.Silence)

main.go

@@ -3,9 +3,9 @@ package main
 import (
 	"path"
 	"strings"
-	"sync"
 	"time"
 
+	"github.com/cloudflare/unsee/alertmanager"
 	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/transform"
 
@@ -17,7 +17,6 @@ import (
 	"github.com/gin-gonic/gin"
 	ginprometheus "github.com/mcuadros/go-gin-prometheus"
 	"github.com/patrickmn/go-cache"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
@@ -31,46 +30,8 @@ var (
 	// If there are requests with the same filter we should respond from cache
 	// rather than do all the filtering every time
 	apiCache *cache.Cache
-
-	// errorLock holds a mutex used to synchronize updates to AlertmanagerError
-	// to avoid any race between readers and writers
-	errorLock = sync.RWMutex{}
-	// alertManagerError holds the description of last error raised when pulling data
-	// from Alertmanager, if there was any error
-	// This error will be returned in UnseeAlertsResponse and presented by Ui
-	alertManagerError string
-
-	metricAlerts = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "unsee_collected_alerts",
-			Help: "Total number of alerts collected from Alertmanager API",
-		},
-		[]string{"state"},
-	)
-	metricAlertGroups = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "unsee_collected_groups",
-			Help: "Total number of alert groups collected from Alertmanager API",
-		},
-	)
-	metricAlertmanagerErrors = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "unsee_alertmanager_errors_total",
-			Help: "Total number of errors encounter when requesting data from Alertmanager API",
-		},
-		[]string{"endpoint"},
-	)
 )
 
-func init() {
-	prometheus.MustRegister(metricAlerts)
-	prometheus.MustRegister(metricAlertGroups)
-	prometheus.MustRegister(metricAlertmanagerErrors)
-
-	metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "alerts"}).Set(0)
-	metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "silences"}).Set(0)
-}
-
 func getViewURL(sub string) string {
 	u := path.Join(config.Config.WebPrefix, sub)
 	if strings.HasSuffix(sub, "/") {
@@ -91,6 +52,18 @@ func setupRouter(router *gin.Engine) {
 	router.GET(getViewURL("/autocomplete.json"), autocomplete)
 }
 
+func setupUpstreams() {
+	for _, s := range config.Config.AlertmanagerURI {
+		z := strings.SplitN(s, ":", 2)
+		if len(z) != 2 {
+			log.Fatalf("Invalid Alertmanager URI '%s', expected format 'name:uri'", s)
+		}
+		name := z[0]
+		uri := z[1]
+		alertmanager.NewAlertmanager(name, uri, config.Config.AlertmanagerTimeout)
+	}
+}
+
 func main() {
 	log.Infof("Version: %s", version)
 
@@ -100,9 +73,11 @@ func main() {
 
 	apiCache = cache.New(cache.NoExpiration, 10*time.Second)
 
+	setupUpstreams()
+
 	// before we start try to fetch data from Alertmanager
 	log.Infof("Initial Alertmanager query, this can delay startup up to %s", 3*config.Config.AlertmanagerTimeout)
-	PullFromAlertmanager()
+	pullFromAlertmanager()
 	log.Info("Done, starting HTTP server")
 
 	// background loop that will fetch updates from Alertmanager

mapper/mapper.go

@@ -2,6 +2,7 @@ package mapper
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/cloudflare/unsee/models"
 )
@@ -15,7 +16,7 @@ var (
 // for a specific range of Alertmanager versions
 type AlertMapper interface {
 	IsSupported(version string) bool
-	GetAlerts() ([]models.AlertGroup, error)
+	GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error)
 }
 
 // RegisterAlertMapper allows to register mapper implementing alert data
@@ -39,7 +40,7 @@ func GetAlertMapper(version string) (AlertMapper, error) {
 type SilenceMapper interface {
 	Release() string
 	IsSupported(version string) bool
-	GetSilences() ([]models.Silence, error)
+	GetSilences(uri string, timeout time.Duration) ([]models.Silence, error)
 }
 
 // RegisterSilenceMapper allows to register mapper implementing silence data

mapper/v04/alerts.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/blang/semver"
-	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/mapper"
 	"github.com/cloudflare/unsee/models"
 	"github.com/cloudflare/unsee/transport"
@@ -60,17 +59,17 @@ func (m AlertMapper) IsSupported(version string) bool {
 
 // GetAlerts will make a request to Alertmanager API and parse the response
 // It will only return alerts or error (if any)
-func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
+func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) {
 	groups := []models.AlertGroup{}
 	receivers := map[string]alertsGroupReceiver{}
 	resp := alertsGroupsAPISchema{}
 
-	url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups")
+	url, err := transport.JoinURL(uri, "api/v1/alerts/groups")
 	if err != nil {
 		return groups, err
 	}
 
-	err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp)
+	err = transport.ReadJSON(url, timeout, &resp)
 	if err != nil {
 		return groups, err
 	}

mapper/v04/silences.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/blang/semver"
-	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/mapper"
 	"github.com/cloudflare/unsee/models"
 	"github.com/cloudflare/unsee/transport"
@@ -56,18 +55,18 @@ func (m SilenceMapper) IsSupported(version string) bool {
 
 // GetSilences will make a request to Alertmanager API and parse the response
 // It will only return silences or error (if any)
-func (m SilenceMapper) GetSilences() ([]models.Silence, error) {
+func (m SilenceMapper) GetSilences(uri string, timeout time.Duration) ([]models.Silence, error) {
 	silences := []models.Silence{}
 	resp := silenceAPISchema{}
 
-	url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences")
+	url, err := transport.JoinURL(uri, "api/v1/silences")
 	if err != nil {
 		return silences, err
 	}
 
 	// Alertmanager 0.4 uses pagination for silences
 	url = fmt.Sprintf("%s?limit=%d", url, math.MaxUint32)
-	err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp)
+	err = transport.ReadJSON(url, timeout, &resp)
 	if err != nil {
 		return silences, err
 	}

mapper/v05/alerts.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/blang/semver"
-	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/mapper"
 	"github.com/cloudflare/unsee/models"
 	"github.com/cloudflare/unsee/transport"
@@ -59,17 +58,17 @@ func (m AlertMapper) IsSupported(version string) bool {
 
 // GetAlerts will make a request to Alertmanager API and parse the response
 // It will only return alerts or error (if any)
-func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
+func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) {
 	groups := []models.AlertGroup{}
 	receivers := map[string]alertsGroupReceiver{}
 	resp := alertsGroupsAPISchema{}
 
-	url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups")
+	url, err := transport.JoinURL(uri, "api/v1/alerts/groups")
 	if err != nil {
 		return groups, err
 	}
 
-	err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp)
+	err = transport.ReadJSON(url, timeout, &resp)
 	if err != nil {
 		return groups, err
 	}

mapper/v05/silences.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/blang/semver"
-	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/mapper"
 	"github.com/cloudflare/unsee/models"
 	"github.com/cloudflare/unsee/transport"
@@ -48,16 +47,16 @@ func (m SilenceMapper) IsSupported(version string) bool {
 
 // GetSilences will make a request to Alertmanager API and parse the response
 // It will only return silences or error (if any)
-func (m SilenceMapper) GetSilences() ([]models.Silence, error) {
+func (m SilenceMapper) GetSilences(uri string, timeout time.Duration) ([]models.Silence, error) {
 	silences := []models.Silence{}
 	resp := silenceAPISchema{}
 
-	url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences")
+	url, err := transport.JoinURL(uri, "api/v1/silences")
 	if err != nil {
 		return silences, err
 	}
 
-	err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp)
+	err = transport.ReadJSON(url, timeout, &resp)
 	if err != nil {
 		return silences, err
 	}

mapper/v061/alerts.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/blang/semver"
-	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/mapper"
 	"github.com/cloudflare/unsee/models"
 	"github.com/cloudflare/unsee/transport"
@@ -61,17 +60,17 @@ func (m AlertMapper) IsSupported(version string) bool {
 
 // GetAlerts will make a request to Alertmanager API and parse the response
 // It will only return alerts or error (if any)
-func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
+func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) {
 	groups := []models.AlertGroup{}
 	receivers := map[string]alertsGroupReceiver{}
 	resp := alertsGroupsAPISchema{}
 
-	url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups")
+	url, err := transport.JoinURL(uri, "api/v1/alerts/groups")
 	if err != nil {
 		return groups, err
 	}
 
-	err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp)
+	err = transport.ReadJSON(url, timeout, &resp)
 	if err != nil {
 		return groups, err
 	}

mapper/v062/alerts.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	"github.com/blang/semver"
-	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/mapper"
 	"github.com/cloudflare/unsee/models"
 	"github.com/cloudflare/unsee/transport"
@@ -65,17 +64,17 @@ func (m AlertMapper) IsSupported(version string) bool {
 
 // GetAlerts will make a request to Alertmanager API and parse the response
 // It will only return alerts or error (if any)
-func (m AlertMapper) GetAlerts() ([]models.AlertGroup, error) {
+func (m AlertMapper) GetAlerts(uri string, timeout time.Duration) ([]models.AlertGroup, error) {
 	groups := []models.AlertGroup{}
 	receivers := map[string]alertsGroupReceiver{}
 	resp := alertsGroupsAPISchema{}
 
-	url, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/alerts/groups")
+	url, err := transport.JoinURL(uri, "api/v1/alerts/groups")
 	if err != nil {
 		return groups, err
 	}
 
-	err = transport.ReadJSON(url, config.Config.AlertmanagerTimeout, &resp)
+	err = transport.ReadJSON(url, timeout, &resp)
 	if err != nil {
 		return groups, err
 	}

models/models.go

@@ -1,6 +1,11 @@
 package models
 
-import "time"
+import (
+	"crypto/sha1"
+	"fmt"
+	"io"
+	"time"
+)
 
 // Silence is vanilla silence + some additional attributes
 // Unsee adds JIRA support, it can extract JIRA IDs from comments
@@ -40,6 +45,15 @@ var AlertStateList = []string{
 	AlertStateSuppressed,
 }
 
+// AlertmanagerUpstream describes the Alertmanager instance alert was collected
+// from
+type AlertmanagerUpstream struct {
+	Name string `json:"name"`
+	URI  string `json:"uri"`
+	// all silences matching current alert in this upstream
+	Silences map[string]Silence `json:"silences"`
+}
+
 // Alert is vanilla alert + some additional attributes
 // unsee extends an alert object with:
 // * Links map, it's generated from annotations if annotation value is an url
@@ -56,9 +70,11 @@ type Alert struct {
 	SilencedBy   []string          `json:"silencedBy"`
 	InhibitedBy  []string          `json:"inhibitedBy"`
 	// unsee fields
-	Receiver    string            `json:"receiver"`
-	Links       map[string]string `json:"links"`
-	Fingerprint string            `json:"-"`
+	Alertmanager []AlertmanagerUpstream `json:"alertmanager"`
+	Receiver     string                 `json:"receiver"`
+	Links        map[string]string      `json:"links"`
+	ID           string                 `json:"-"`
+	Fingerprint  string                 `json:"-"`
 }
 
 // IsSilenced will return true if alert should be considered silenced
@@ -109,6 +125,15 @@ type AlertGroup struct {
 	StateCount map[string]int    `json:"stateCount"`
 }
 
+// ContentFingerprint is a checksum of all alerts in the group
+func (ag AlertGroup) ContentFingerprint() string {
+	h := sha1.New()
+	for _, alert := range ag.Alerts {
+		io.WriteString(h, alert.Fingerprint)
+	}
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
 // Filter holds returned data on any filter passed by the user as part of the query
 type Filter struct {
 	Text    string `json:"text"`
@@ -139,15 +164,14 @@ type LabelsCountMap map[string]map[string]int
 
 // AlertsResponse is the structure of JSON response UI will use to get alert data
 type AlertsResponse struct {
-	Status      string             `json:"status"`
-	Error       string             `json:"error,omitempty"`
-	Timestamp   string             `json:"timestamp"`
-	Version     string             `json:"version"`
-	AlertGroups []AlertGroup       `json:"groups"`
-	Silences    map[string]Silence `json:"silences"`
-	Colors      LabelsColorMap     `json:"colors"`
-	Filters     []Filter           `json:"filters"`
-	Counters    LabelsCountMap     `json:"counters"`
+	Status      string         `json:"status"`
+	Error       string         `json:"error,omitempty"`
+	Timestamp   string         `json:"timestamp"`
+	Version     string         `json:"version"`
+	AlertGroups []AlertGroup   `json:"groups"`
+	Colors      LabelsColorMap `json:"colors"`
+	Filters     []Filter       `json:"filters"`
+	Counters    LabelsCountMap `json:"counters"`
 }
 
 // Autocomplete is the structure of autocomplete object for filter hints

store/store.go

@@ -1,44 +0,0 @@
-package store
-
-import (
-	"sync"
-
-	"github.com/cloudflare/unsee/models"
-)
-
-type dataStore struct {
-	Lock         sync.RWMutex
-	Groups       []models.AlertGroup
-	Silences     map[string]models.Silence
-	Colors       models.LabelsColorMap
-	Autocomplete []models.Autocomplete
-}
-
-// Store will keep all Alertmanager data we collect
-var Store = dataStore{}
-
-// GetSilence returns silence data for specific silence id or nil if not found
-func (ds *dataStore) GetSilence(s string) *models.Silence {
-	ds.Lock.RLock()
-	defer ds.Lock.RUnlock()
-	if silence, found := ds.Silences[s]; found {
-		return &silence
-	}
-	return nil
-}
-
-// SetSilences allows to update silence list stored internally
-func (ds *dataStore) SetSilences(s map[string]models.Silence) {
-	ds.Lock.Lock()
-	defer ds.Lock.Unlock()
-	ds.Silences = s
-}
-
-// Update will lock the store and update internal data
-func (ds *dataStore) Update(groups []models.AlertGroup, colors models.LabelsColorMap, autocomplete []models.Autocomplete) {
-	ds.Lock.Lock()
-	defer ds.Lock.Unlock()
-	ds.Groups = groups
-	ds.Colors = colors
-	ds.Autocomplete = autocomplete
-}

store/store_test.go

@@ -1,57 +0,0 @@
-package store_test
-
-import (
-	"testing"
-
-	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/store"
-)
-
-type silenceTest struct {
-	silences  map[string]models.Silence
-	silenceID string
-	found     bool
-}
-
-var silenceTests = []silenceTest{
-	silenceTest{
-		silences: map[string]models.Silence{
-			"1": models.Silence{},
-		},
-		silenceID: "1",
-		found:     true,
-	},
-	silenceTest{
-		silences: map[string]models.Silence{
-			"1": models.Silence{},
-			"2": models.Silence{},
-			"3": models.Silence{},
-		},
-		silenceID: "2",
-		found:     true,
-	},
-	silenceTest{
-		silences:  map[string]models.Silence{},
-		silenceID: "1",
-		found:     false,
-	},
-	silenceTest{
-		silences: map[string]models.Silence{
-			"2": models.Silence{},
-			"3": models.Silence{},
-		},
-		silenceID: "1",
-		found:     false,
-	},
-}
-
-func TestSilences(t *testing.T) {
-	for _, testCase := range silenceTests {
-		store.Store.SetSilences(testCase.silences)
-		silence := store.Store.GetSilence(testCase.silenceID)
-		found := silence != nil
-		if found != testCase.found {
-			t.Errorf("GetSilence('%s') returned %v, %v was expected", testCase.silenceID, found, testCase.found)
-		}
-	}
-}

timer.go

@@ -1,149 +1,37 @@
 package main
 
 import (
-	"crypto/sha1"
-	"fmt"
-	"io"
 	"runtime"
-	"sort"
+	"sync"
 
 	"github.com/cloudflare/unsee/alertmanager"
-	"github.com/cloudflare/unsee/config"
-	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/store"
-	"github.com/cloudflare/unsee/transform"
-	"github.com/cnf/structhash"
 
 	log "github.com/Sirupsen/logrus"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
-// PullFromAlertmanager will try to fetch latest alerts and silences
-// from Alertmanager API, it's called by Ticker timer
-func PullFromAlertmanager() {
+func pullFromAlertmanager() {
 	// always flush cache once we're done
 	defer apiCache.Flush()
 
 	log.Info("Pulling latest alerts and silences from Alertmanager")
-	v := alertmanager.GetVersion()
 
-	silences, err := alertmanager.GetSilences(v)
-	if err != nil {
-		log.Error(err.Error())
-		errorLock.Lock()
-		alertManagerError = err.Error()
-		errorLock.Unlock()
-		metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "silences"}).Inc()
-		return
-	}
+	upstreams := alertmanager.GetAlertmanagers()
+	wg := sync.WaitGroup{}
+	wg.Add(len(upstreams))
 
-	alertGroups, err := alertmanager.GetAlerts(v)
-	if err != nil {
-		log.Error(err.Error())
-		errorLock.Lock()
-		alertManagerError = err.Error()
-		errorLock.Unlock()
-		metricAlertmanagerErrors.With(prometheus.Labels{"endpoint": "alerts"}).Inc()
-		return
-	}
-
-	log.Infof("Detecting JIRA links in silences (%d)", len(silences))
-	silenceStore := make(map[string]models.Silence)
-	for _, silence := range silences {
-		silence.JiraID, silence.JiraURL = transform.DetectJIRAs(&silence)
-		silenceStore[silence.ID] = silence
-	}
-
-	log.Infof("Updating list of stored silences (%d)", len(silenceStore))
-	store.Store.SetSilences(silenceStore)
-
-	alertStore := []models.AlertGroup{}
-	colorStore := make(models.LabelsColorMap)
-
-	acMap := map[string]models.Autocomplete{}
-
-	for _, state := range models.AlertStateList {
-		metricAlerts.With(prometheus.Labels{"state": state}).Set(0)
-	}
-
-	log.Infof("Deduplicating alert groups (%d)", len(alertGroups))
-	uniqueGroups := map[string]models.AlertGroup{}
-	uniqueAlerts := map[string]map[string]models.Alert{}
-	for _, ag := range alertGroups {
-		agIDHasher := sha1.New()
-		io.WriteString(agIDHasher, ag.Receiver)
-		io.WriteString(agIDHasher, fmt.Sprintf("%x", structhash.Sha1(ag.Labels, 1)))
-		agID := fmt.Sprintf("%x", agIDHasher.Sum(nil))
-		if _, found := uniqueGroups[agID]; !found {
-			uniqueGroups[agID] = models.AlertGroup{
-				Receiver: ag.Receiver,
-				Labels:   ag.Labels,
-				ID:       agID,
+	for _, upstream := range upstreams {
+		go func(am *alertmanager.Alertmanager) {
+			log.Infof("[%s] Collecting alerts and silences", am.Name)
+			err := am.Pull()
+			if err != nil {
+				log.Errorf("[%s] %s", am.Name, err)
 			}
-		}
-		for _, alert := range ag.Alerts {
-			// generate alert fingerprint from a raw, unaltered alert object
-			aID := fmt.Sprintf("%x", structhash.Sha1(alert, 1))
-			if _, found := uniqueAlerts[agID]; !found {
-				uniqueAlerts[agID] = map[string]models.Alert{}
-			}
-			if _, found := uniqueAlerts[agID][aID]; !found {
-				alert.Fingerprint = aID
-				uniqueAlerts[agID][aID] = alert
-			}
-		}
+			wg.Done()
+		}(upstream)
 	}
 
-	log.Infof("Processing unique alert groups (%d)", len(uniqueGroups))
-	for _, ag := range uniqueGroups {
-		// used to generate group content hash
-		agHasher := sha1.New()
+	wg.Wait()
 
-		alerts := models.AlertList{}
-		for _, alert := range uniqueAlerts[ag.ID] {
-
-			alert.Annotations, alert.Links = transform.DetectLinks(alert.Annotations)
-			alert.Labels = transform.StripLables(config.Config.StripLabels, alert.Labels)
-
-			io.WriteString(agHasher, alert.Fingerprint) // alert group hasher
-
-			transform.ColorLabel(colorStore, "@receiver", alert.Receiver)
-			for k, v := range alert.Labels {
-				transform.ColorLabel(colorStore, k, v)
-			}
-			alerts = append(alerts, alert)
-
-			// update internal metrics
-			metricAlerts.With(prometheus.Labels{"state": alert.State}).Inc()
-		}
-
-		for _, hint := range transform.BuildAutocomplete(alerts) {
-			acMap[hint.Value] = hint
-		}
-
-		sort.Sort(&alerts)
-		ag.Alerts = alerts
-
-		// Hash is a checksum of all alerts, used to tell when any alert in the group changed
-		ag.Hash = fmt.Sprintf("%x", agHasher.Sum(nil))
-
-		alertStore = append(alertStore, ag)
-	}
-
-	log.Infof("Merging autocomplete data (%d)", len(acMap))
-	acStore := []models.Autocomplete{}
-	for _, hint := range acMap {
-		acStore = append(acStore, hint)
-	}
-
-	errorLock.Lock()
-	alertManagerError = ""
-	errorLock.Unlock()
-
-	metricAlertGroups.Set(float64(len(alertStore)))
-
-	log.Infof("Updating list of stored alert groups (%d)", len(alertStore))
-	store.Store.Update(alertStore, colorStore, acStore)
 	log.Info("Pull completed")
 	runtime.GC()
 }
@@ -153,7 +41,7 @@ func Tick() {
 	for {
 		select {
 		case <-ticker.C:
-			PullFromAlertmanager()
+			pullFromAlertmanager()
 		}
 	}
 }

views.go

@@ -1,19 +1,15 @@
 package main
 
 import (
-	"crypto/sha1"
 	"encoding/json"
-	"fmt"
-	"io"
 	"net/http"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/cloudflare/unsee/alertmanager"
 	"github.com/cloudflare/unsee/config"
 	"github.com/cloudflare/unsee/models"
-	"github.com/cloudflare/unsee/store"
-	"github.com/cloudflare/unsee/transport"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/gin-gonic/gin"
@@ -51,10 +47,11 @@ func index(c *gin.Context) {
 		defaultUsed = false
 	}
 
-	silencesAPI, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences")
-	if err != nil {
-		log.Errorf("Can't generate silences API URL: %s", err)
-	}
+	// FIXME
+	//silencesAPI, err := transport.JoinURL(config.Config.AlertmanagerURI, "api/v1/silences")
+	//if err != nil {
+	//	log.Errorf("Can't generate silences API URL: %s", err)
+	//}
 
 	c.HTML(http.StatusOK, "templates/index.html", gin.H{
 		"Version":           version,
@@ -67,7 +64,7 @@ func index(c *gin.Context) {
 		"DefaultUsed":       defaultUsed,
 		"StaticColorLabels": strings.Join(config.Config.ColorLabelsStatic, " "),
 		"WebPrefix":         config.Config.WebPrefix,
-		"SilencesApi":       silencesAPI,
+		"SilencesApi":       "FIXME",
 	})
 
 	log.Infof("[%s] %s %s took %s", c.ClientIP(), c.Request.Method, c.Request.RequestURI, time.Since(start))
@@ -103,9 +100,7 @@ func alerts(c *gin.Context) {
 	resp.Version = version
 
 	// update error field, needs a lock
-	errorLock.RLock()
-	resp.Error = string(alertManagerError)
-	errorLock.RUnlock()
+	// FIXME resp.Error = string(alertManagerError)
 
 	if resp.Error != "" {
 		apiCache.Flush()
@@ -127,13 +122,14 @@ func alerts(c *gin.Context) {
 
 	// set pointers for data store objects, need a lock until end of view is reached
 	alerts := []models.AlertGroup{}
-	silences := map[string]models.Silence{}
 	colors := models.LabelsColorMap{}
 	counters := models.LabelsCountMap{}
-	store.Store.Lock.RLock()
+
+	dedupedAlerts := alertmanager.DedupAlerts()
+	dedupedColors := alertmanager.DedupColors()
 
 	var matches int
-	for _, ag := range store.Store.Groups {
+	for _, ag := range dedupedAlerts {
 		agCopy := models.AlertGroup{
 			ID:         ag.ID,
 			Receiver:   ag.Receiver,
@@ -144,7 +140,6 @@ func alerts(c *gin.Context) {
 		for _, s := range models.AlertStateList {
 			agCopy.StateCount[s] = 0
 		}
-		h := sha1.New()
 
 		for _, alert := range ag.Alerts {
 			results := []bool{}
@@ -159,25 +154,11 @@ func alerts(c *gin.Context) {
 			if !validFilters || (boolInSlice(results, true) && !boolInSlice(results, false)) {
 				matches++
 				agCopy.Alerts = append(agCopy.Alerts, alert)
-				aj, err := json.Marshal(alert)
-				if err != nil {
-					log.Error(err.Error())
-					panic(err.Error())
-				}
-				io.WriteString(h, string(aj))
-
-				if alert.IsSilenced() {
-					for _, silenceID := range alert.SilencedBy {
-						if silence := store.Store.GetSilence(silenceID); silence != nil {
-							silences[silenceID] = *silence
-						}
-					}
-				}
 
 				countLabel(counters, "@state", alert.State)
 
 				countLabel(counters, "@receiver", alert.Receiver)
-				if ck, foundKey := store.Store.Colors["@receiver"]; foundKey {
+				if ck, foundKey := dedupedColors["@receiver"]; foundKey {
 					if cv, foundVal := ck[alert.Receiver]; foundVal {
 						if _, found := colors["@receiver"]; !found {
 							colors["@receiver"] = map[string]models.LabelColors{}
@@ -189,7 +170,7 @@ func alerts(c *gin.Context) {
 				agCopy.StateCount[alert.State]++
 
 				for key, value := range alert.Labels {
-					if keyMap, foundKey := store.Store.Colors[key]; foundKey {
+					if keyMap, foundKey := dedupedColors[key]; foundKey {
 						if color, foundColor := keyMap[value]; foundColor {
 							if _, found := colors[key]; !found {
 								colors[key] = map[string]models.LabelColors{}
@@ -203,14 +184,13 @@ func alerts(c *gin.Context) {
 		}
 
 		if len(agCopy.Alerts) > 0 {
-			agCopy.Hash = fmt.Sprintf("%x", h.Sum(nil))
+			agCopy.Hash = agCopy.ContentFingerprint()
 			alerts = append(alerts, agCopy)
 		}
 
 	}
 
 	resp.AlertGroups = alerts
-	resp.Silences = silences
 	resp.Colors = colors
 	resp.Counters = counters
 
@@ -230,7 +210,6 @@ func alerts(c *gin.Context) {
 		panic(err)
 	}
 	apiCache.Set(cacheKey, data, -1)
-	store.Store.Lock.RUnlock()
 
 	c.Data(http.StatusOK, gin.MIMEJSON, data.([]byte))
 	logAlertsView(c, "MIS", time.Since(start))
@@ -264,8 +243,9 @@ func autocomplete(c *gin.Context) {
 
 	acData := sort.StringSlice{}
 
-	store.Store.Lock.RLock()
-	for _, hint := range store.Store.Autocomplete {
+	dedupedAutocomplete := alertmanager.DedupAutocomplete()
+
+	for _, hint := range dedupedAutocomplete {
 		if strings.HasPrefix(strings.ToLower(hint.Value), strings.ToLower(term)) {
 			acData = append(acData, hint.Value)
 		} else {
@@ -276,7 +256,6 @@ func autocomplete(c *gin.Context) {
 			}
 		}
 	}
-	store.Store.Lock.RUnlock()
 
 	sort.Sort(sort.Reverse(acData))
 	data, err := json.Marshal(acData)

views_test.go

@@ -31,9 +31,10 @@ func stringInSlice(stringArray []string, value string) bool {
 
 func mockConfig() {
 	log.SetLevel(log.ErrorLevel)
-	os.Setenv("ALERTMANAGER_URI", "http://localhost")
+	os.Setenv("ALERTMANAGER_URI", "default:http://localhost")
 	os.Setenv("COLOR_LABELS_UNIQUE", "alertname")
 	config.Config.Read()
+	setupUpstreams()
 }
 
 func ginTestEngine() *gin.Engine {
@@ -108,7 +109,7 @@ func mockAlerts(version string) {
 	mock.RegisterURL("http://localhost/api/v1/silences", version, "silences")
 	mock.RegisterURL("http://localhost/api/v1/alerts/groups", version, "alerts/groups")
 
-	PullFromAlertmanager()
+	pullFromAlertmanager()
 }
 
 func TestAlerts(t *testing.T) {
@@ -126,22 +127,19 @@ func TestAlerts(t *testing.T) {
 		ur := models.AlertsResponse{}
 		json.Unmarshal(resp.Body.Bytes(), &ur)
 		if len(ur.Filters) != 3 {
-			t.Errorf("[%s] No filters in response", version)
+			t.Errorf("[%s] Got %d filter(s) in response, expected %d", version, len(ur.Filters), 3)
 		}
 		if len(ur.Colors) != 1 {
-			t.Errorf("[%s] No colors in response", version)
-		}
-		if len(ur.Silences) != 1 {
-			t.Errorf("[%s] No silences in response", version)
+			t.Errorf("[%s] Got %d color(s) in response, expected %d", version, len(ur.Colors), 1)
 		}
 		if len(ur.AlertGroups) != 1 {
-			t.Errorf("[%s] No alerts in response", version)
+			t.Errorf("[%s] Got %d alert(s) in response, expected %d", version, len(ur.AlertGroups), 1)
 		}
 		if ur.Version == "" {
-			t.Errorf("[%s] No version in response", version)
+			t.Errorf("[%s] Empty version in response", version)
 		}
 		if ur.Timestamp == "" {
-			t.Errorf("[%s] No timestamp in response", version)
+			t.Errorf("[%s] Empty timestamp in response", version)
 		}
 		if ur.Error != "" {
 			t.Errorf("[%s] Error in response: %s", version, ur.Error)