From 6c7ba5551cef0076cadccfa85b07213f1826ef5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Thu, 5 Jul 2018 13:25:23 +0200 Subject: [PATCH] refactor(api): move silences into a dedicated key in API response A single silence can supress multiple alerts so to reduce response size and the amount of data react UI needs to track move it out of alert instance (which will deduplicate it) and on each alert only store a reference. In the API return all silences as a global dict and each alert only contains the ID of silences suppressing it. --- api_test.go | 242 +++++++++++++------------------- internal/alertmanager/models.go | 32 +++-- internal/filters/filter_test.go | 5 +- internal/models/alertmanager.go | 7 +- internal/models/api.go | 21 +-- views.go | 15 ++ 6 files changed, 157 insertions(+), 165 deletions(-) diff --git a/api_test.go b/api_test.go index 1c4653bac..2f42c959b 100644 --- a/api_test.go +++ b/api_test.go @@ -42,10 +42,10 @@ var groupTests = []groupTest{ State: models.AlertStateActive, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Receiver: "by-name", @@ -73,10 +73,10 @@ var groupTests = []groupTest{ }, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -106,10 +106,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -122,10 +122,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -138,10 +138,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -178,16 +178,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateSuppressed, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{ - ID: "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d", - CreatedBy: "john@example.com", - Comment: "Silenced Host_Down alerts in the dev cluster", - }, - }, + Name: "default", + State: models.AlertStateSuppressed, + Source: "localhost/prometheus", + SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"}, }, }, Labels: map[string]string{ @@ -200,19 +194,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateSuppressed, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced Host_Down alerts in the dev cluster", - }, - "378eaa69-097d-41c4-a8c2-fe6568c3abfc": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced server7", - }, - }, + Name: "default", + State: models.AlertStateSuppressed, + Source: "localhost/prometheus", + SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d", "378eaa69-097d-41c4-a8c2-fe6568c3abfc"}, }, }, Labels: map[string]string{ @@ -225,15 +210,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateSuppressed, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced Host_Down alerts in the dev cluster", - }, - }, + Name: "default", + State: models.AlertStateSuppressed, + Source: "localhost/prometheus", + SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"}, }, }, Labels: map[string]string{ @@ -276,10 +256,10 @@ var groupTests = []groupTest{ State: models.AlertStateActive, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Receiver: "by-name", @@ -293,10 +273,10 @@ var groupTests = []groupTest{ State: models.AlertStateActive, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Receiver: "by-name", @@ -310,10 +290,10 @@ var groupTests = []groupTest{ State: models.AlertStateActive, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Receiver: "by-name", @@ -327,10 +307,10 @@ var groupTests = []groupTest{ State: models.AlertStateActive, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Receiver: "by-name", @@ -344,10 +324,10 @@ var groupTests = []groupTest{ State: models.AlertStateActive, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Receiver: "by-name", @@ -361,15 +341,10 @@ var groupTests = []groupTest{ State: models.AlertStateSuppressed, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateSuppressed, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced Host_Down alerts in the dev cluster", - }, - }, + Name: "default", + State: models.AlertStateSuppressed, + Source: "localhost/prometheus", + SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"}, }, }, Receiver: "by-name", @@ -383,19 +358,10 @@ var groupTests = []groupTest{ State: models.AlertStateSuppressed, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateSuppressed, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced Host_Down alerts in the dev cluster", - }, - "378eaa69-097d-41c4-a8c2-fe6568c3abfc": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced server7", - }, - }, + Name: "default", + State: models.AlertStateSuppressed, + Source: "localhost/prometheus", + SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d", "378eaa69-097d-41c4-a8c2-fe6568c3abfc"}, }, }, Receiver: "by-name", @@ -409,15 +375,10 @@ var groupTests = []groupTest{ State: models.AlertStateSuppressed, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateSuppressed, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced Host_Down alerts in the dev cluster", - }, - }, + Name: "default", + State: models.AlertStateSuppressed, + Source: "localhost/prometheus", + SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"}, }, }, Receiver: "by-name", @@ -453,10 +414,10 @@ var groupTests = []groupTest{ }, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -488,10 +449,10 @@ var groupTests = []groupTest{ }, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -504,10 +465,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -549,12 +510,6 @@ var groupTests = []groupTest{ Name: "default", State: models.AlertStateSuppressed, Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "0804764c-6163-4c64-b0a9-08feebe2db4b": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced instance", - }, - }, }, }, Labels: map[string]string{ @@ -567,10 +522,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -610,10 +565,10 @@ var groupTests = []groupTest{ }, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -647,15 +602,10 @@ var groupTests = []groupTest{ }, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateSuppressed, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{ - "0804764c-6163-4c64-b0a9-08feebe2db4b": models.Silence{ - CreatedBy: "john@example.com", - Comment: "Silenced instance", - }, - }, + Name: "default", + State: models.AlertStateSuppressed, + Source: "localhost/prometheus", + SilencedBy: []string{"0804764c-6163-4c64-b0a9-08feebe2db4b"}, }, }, Labels: map[string]string{ @@ -668,10 +618,10 @@ var groupTests = []groupTest{ Annotations: models.Annotations{}, Alertmanager: []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: "default", - State: models.AlertStateActive, - Source: "localhost/prometheus", - Silences: map[string]models.Silence{}, + Name: "default", + State: models.AlertStateActive, + Source: "localhost/prometheus", + SilencedBy: []string{}, }, }, Labels: map[string]string{ @@ -913,6 +863,14 @@ func TestVerifyAllGroups(t *testing.T) { version, testCase.receiver, testCase.labels) } } + + am, foundAM := ur.Silences["default"] + if !foundAM { + t.Errorf("[%s] Alertmanager 'default' missing from silences", version) + } else if len(am) == 0 { + t.Errorf("[%s] Silences mismatch, expected >0 but got %d", version, len(am)) + } + for key, expectedCounts := range countsMap { gotCounts, foundCounts := ur.Counters[key] if !foundCounts { diff --git a/internal/alertmanager/models.go b/internal/alertmanager/models.go index 9b1ae7d2f..ea0776c2f 100644 --- a/internal/alertmanager/models.go +++ b/internal/alertmanager/models.go @@ -235,22 +235,24 @@ func (am *Alertmanager) pullAlerts(version string) error { alerts := models.AlertList{} for _, alert := range uniqueAlerts[ag.ID] { - silences := map[string]models.Silence{} + silences := map[string]*models.Silence{} for _, silenceID := range alert.SilencedBy { silence, err := am.SilenceByID(silenceID) if err == nil { - silences[silenceID] = silence + silences[silenceID] = &silence } } + alert.Alertmanager = []models.AlertmanagerInstance{ models.AlertmanagerInstance{ - Name: am.Name, - URI: am.publicURI(), - State: alert.State, - StartsAt: alert.StartsAt, - EndsAt: alert.EndsAt, - Source: alert.GeneratorURL, - Silences: silences, + Name: am.Name, + URI: am.publicURI(), + State: alert.State, + StartsAt: alert.StartsAt, + EndsAt: alert.EndsAt, + Source: alert.GeneratorURL, + Silences: silences, + SilencedBy: alert.SilencedBy, }, } @@ -327,6 +329,18 @@ func (am *Alertmanager) Alerts() []models.AlertGroup { return alerts } +// Silences returns a copy of all silences +func (am *Alertmanager) Silences() map[string]models.Silence { + am.lock.RLock() + defer am.lock.RUnlock() + + silences := map[string]models.Silence{} + for id, silence := range am.silences { + silences[id] = silence + } + return silences +} + // SilenceByID allows to query for a silence by it's ID, returns error if not found func (am *Alertmanager) SilenceByID(id string) (models.Silence, error) { am.lock.RLock() diff --git a/internal/filters/filter_test.go b/internal/filters/filter_test.go index 5f7543dec..83827ad2b 100644 --- a/internal/filters/filter_test.go +++ b/internal/filters/filter_test.go @@ -496,9 +496,10 @@ func TestFilters(t *testing.T) { models.AlertmanagerInstance{ Name: am.Name, URI: am.URI, - Silences: map[string]models.Silence{ - ft.Silence.ID: ft.Silence, + Silences: map[string]*models.Silence{ + ft.Silence.ID: &ft.Silence, }, + SilencedBy: []string{ft.Silence.ID}, }, } } diff --git a/internal/models/alertmanager.go b/internal/models/alertmanager.go index 392f39a78..f95c75244 100644 --- a/internal/models/alertmanager.go +++ b/internal/models/alertmanager.go @@ -15,8 +15,11 @@ type AlertmanagerInstance struct { EndsAt time.Time `json:"endsAt"` // Source links to alert source for given alertmanager instance Source string `json:"source"` - // all silences matching current alert in this upstream - Silences map[string]Silence `json:"silences"` + // all silences matching current alert in this upstream, we don't export this + // in api responses, this is used internally + Silences map[string]*Silence `json:"-"` + // export list of silenced IDs in api response + SilencedBy []string `json:"silencedBy"` // TODO also export InhibitedBy here if it ever becomes needed } diff --git a/internal/models/api.go b/internal/models/api.go index 25b5f0af8..c7c2f88aa 100644 --- a/internal/models/api.go +++ b/internal/models/api.go @@ -160,16 +160,17 @@ func (ag *APIAlertGroup) DedupSharedMaps() { // AlertsResponse is the structure of JSON response UI will use to get alert data type AlertsResponse struct { - Status string `json:"status"` - Timestamp string `json:"timestamp"` - Version string `json:"version"` - Upstreams AlertmanagerAPISummary `json:"upstreams"` - AlertGroups map[string]APIAlertGroup `json:"groups"` - TotalAlerts int `json:"totalAlerts"` - Colors LabelsColorMap `json:"colors"` - Filters []Filter `json:"filters"` - Counters LabelsCountMap `json:"counters"` - StaticColorLabels []string `json:"staticColorLabels"` + Status string `json:"status"` + Timestamp string `json:"timestamp"` + Version string `json:"version"` + Upstreams AlertmanagerAPISummary `json:"upstreams"` + Silences map[string]map[string]Silence `json:"silences"` + AlertGroups map[string]APIAlertGroup `json:"groups"` + TotalAlerts int `json:"totalAlerts"` + Colors LabelsColorMap `json:"colors"` + Filters []Filter `json:"filters"` + Counters LabelsCountMap `json:"counters"` + StaticColorLabels []string `json:"staticColorLabels"` } // Autocomplete is the structure of autocomplete object for filter hints diff --git a/views.go b/views.go index 68482bbee..b65466222 100644 --- a/views.go +++ b/views.go @@ -76,6 +76,11 @@ func alerts(c *gin.Context) { dedupedAlerts := alertmanager.DedupAlerts() dedupedColors := alertmanager.DedupColors() + silences := map[string]map[string]models.Silence{} + for _, am := range alertmanager.GetAlertmanagers() { + silences[am.Name] = map[string]models.Silence{} + } + var matches int for _, ag := range dedupedAlerts { agCopy := models.AlertGroup{ @@ -137,6 +142,15 @@ func alerts(c *gin.Context) { } if len(agCopy.Alerts) > 0 { + for _, alert := range agCopy.Alerts { + if alert.IsSilenced() { + for _, am := range alert.Alertmanager { + for _, silence := range am.Silences { + silences[am.Name][silence.ID] = *silence + } + } + } + } agCopy.Hash = agCopy.ContentFingerprint() apiAG := models.APIAlertGroup{AlertGroup: agCopy} apiAG.DedupSharedMaps() @@ -147,6 +161,7 @@ func alerts(c *gin.Context) { } resp.AlertGroups = alerts + resp.Silences = silences resp.Colors = colors resp.Counters = counters