github/karma
1
0
Fork 0
mirror of https://github.com/prymitive/karma synced 2026-08-23 11:56:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(api): move silences into a dedicated key in API response

Browse Source 
A single silence can supress multiple alerts so to reduce response size and the amount of data react UI needs to track move it out of alert instance (which will deduplicate it) and on each alert only store a reference. In the API return all silences as a global dict and each alert only contains the ID of silences suppressing it.
This commit is contained in:
Łukasz Mierzwa
2018-07-19 23:38:11 +02:00
parent 79ff668019
commit 6c7ba5551c
6 changed files with 157 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api_test.go
+100 -142
View File
@@ -42,10 +42,10 @@ var groupTests = []groupTest{
State: models.AlertStateActive,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Receiver: "by-name",
@@ -73,10 +73,10 @@ var groupTests = []groupTest{
},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -106,10 +106,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -122,10 +122,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -138,10 +138,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -178,16 +178,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{
ID: "168f139d-77e4-41d6-afb5-8fe2cfd0cc9d",
CreatedBy: "john@example.com",
Comment: "Silenced Host_Down alerts in the dev cluster",
},
},
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"},
},
},
Labels: map[string]string{
@@ -200,19 +194,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced Host_Down alerts in the dev cluster",
},
"378eaa69-097d-41c4-a8c2-fe6568c3abfc": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced server7",
},
},
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d", "378eaa69-097d-41c4-a8c2-fe6568c3abfc"},
},
},
Labels: map[string]string{
@@ -225,15 +210,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced Host_Down alerts in the dev cluster",
},
},
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"},
},
},
Labels: map[string]string{
@@ -276,10 +256,10 @@ var groupTests = []groupTest{
State: models.AlertStateActive,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Receiver: "by-name",
@@ -293,10 +273,10 @@ var groupTests = []groupTest{
State: models.AlertStateActive,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Receiver: "by-name",
@@ -310,10 +290,10 @@ var groupTests = []groupTest{
State: models.AlertStateActive,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Receiver: "by-name",
@@ -327,10 +307,10 @@ var groupTests = []groupTest{
State: models.AlertStateActive,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Receiver: "by-name",
@@ -344,10 +324,10 @@ var groupTests = []groupTest{
State: models.AlertStateActive,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Receiver: "by-name",
@@ -361,15 +341,10 @@ var groupTests = []groupTest{
State: models.AlertStateSuppressed,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced Host_Down alerts in the dev cluster",
},
},
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"},
},
},
Receiver: "by-name",
@@ -383,19 +358,10 @@ var groupTests = []groupTest{
State: models.AlertStateSuppressed,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced Host_Down alerts in the dev cluster",
},
"378eaa69-097d-41c4-a8c2-fe6568c3abfc": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced server7",
},
},
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d", "378eaa69-097d-41c4-a8c2-fe6568c3abfc"},
},
},
Receiver: "by-name",
@@ -409,15 +375,10 @@ var groupTests = []groupTest{
State: models.AlertStateSuppressed,
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced Host_Down alerts in the dev cluster",
},
},
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
SilencedBy: []string{"168f139d-77e4-41d6-afb5-8fe2cfd0cc9d"},
},
},
Receiver: "by-name",
@@ -453,10 +414,10 @@ var groupTests = []groupTest{
},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -488,10 +449,10 @@ var groupTests = []groupTest{
},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -504,10 +465,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -549,12 +510,6 @@ var groupTests = []groupTest{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"0804764c-6163-4c64-b0a9-08feebe2db4b": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced instance",
},
},
},
},
Labels: map[string]string{
@@ -567,10 +522,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -610,10 +565,10 @@ var groupTests = []groupTest{
},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -647,15 +602,10 @@ var groupTests = []groupTest{
},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{
"0804764c-6163-4c64-b0a9-08feebe2db4b": models.Silence{
CreatedBy: "john@example.com",
Comment: "Silenced instance",
},
},
Name: "default",
State: models.AlertStateSuppressed,
Source: "localhost/prometheus",
SilencedBy: []string{"0804764c-6163-4c64-b0a9-08feebe2db4b"},
},
},
Labels: map[string]string{
@@ -668,10 +618,10 @@ var groupTests = []groupTest{
Annotations: models.Annotations{},
Alertmanager: []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
Silences: map[string]models.Silence{},
Name: "default",
State: models.AlertStateActive,
Source: "localhost/prometheus",
SilencedBy: []string{},
},
},
Labels: map[string]string{
@@ -913,6 +863,14 @@ func TestVerifyAllGroups(t *testing.T) {
version, testCase.receiver, testCase.labels)
}
}
am, foundAM := ur.Silences["default"]
if !foundAM {
t.Errorf("[%s] Alertmanager 'default' missing from silences", version)
} else if len(am) == 0 {
t.Errorf("[%s] Silences mismatch, expected >0 but got %d", version, len(am))
}
for key, expectedCounts := range countsMap {
gotCounts, foundCounts := ur.Counters[key]
if !foundCounts {
internal/alertmanager/models.go
+23 -9
View File
@@ -235,22 +235,24 @@ func (am *Alertmanager) pullAlerts(version string) error {
alerts := models.AlertList{}
for _, alert := range uniqueAlerts[ag.ID] {
silences := map[string]models.Silence{}
silences := map[string]*models.Silence{}
for _, silenceID := range alert.SilencedBy {
silence, err := am.SilenceByID(silenceID)
if err == nil {
silences[silenceID] = silence
silences[silenceID] = &silence
}
}
alert.Alertmanager = []models.AlertmanagerInstance{
models.AlertmanagerInstance{
Name: am.Name,
URI: am.publicURI(),
State: alert.State,
StartsAt: alert.StartsAt,
EndsAt: alert.EndsAt,
Source: alert.GeneratorURL,
Silences: silences,
Name: am.Name,
URI: am.publicURI(),
State: alert.State,
StartsAt: alert.StartsAt,
EndsAt: alert.EndsAt,
Source: alert.GeneratorURL,
Silences: silences,
SilencedBy: alert.SilencedBy,
},
}
@@ -327,6 +329,18 @@ func (am *Alertmanager) Alerts() []models.AlertGroup {
return alerts
}
// Silences returns a copy of all silences
func (am *Alertmanager) Silences() map[string]models.Silence {
am.lock.RLock()
defer am.lock.RUnlock()
silences := map[string]models.Silence{}
for id, silence := range am.silences {
silences[id] = silence
}
return silences
}
// SilenceByID allows to query for a silence by it's ID, returns error if not found
func (am *Alertmanager) SilenceByID(id string) (models.Silence, error) {
am.lock.RLock()
internal/filters/filter_test.go
+3 -2
View File
@@ -496,9 +496,10 @@ func TestFilters(t *testing.T) {
models.AlertmanagerInstance{
Name: am.Name,
URI: am.URI,
Silences: map[string]models.Silence{
ft.Silence.ID: ft.Silence,
Silences: map[string]*models.Silence{
ft.Silence.ID: &ft.Silence,
},
SilencedBy: []string{ft.Silence.ID},
},
}
}
internal/models/alertmanager.go
+5 -2
View File
@@ -15,8 +15,11 @@ type AlertmanagerInstance struct {
EndsAt time.Time `json:"endsAt"`
// Source links to alert source for given alertmanager instance
Source string `json:"source"`
// all silences matching current alert in this upstream
Silences map[string]Silence `json:"silences"`
// all silences matching current alert in this upstream, we don't export this
// in api responses, this is used internally
Silences map[string]*Silence `json:"-"`
// export list of silenced IDs in api response
SilencedBy []string `json:"silencedBy"`
// TODO also export InhibitedBy here if it ever becomes needed
}
internal/models/api.go
+11 -10
View File
@@ -160,16 +160,17 @@ func (ag *APIAlertGroup) DedupSharedMaps() {
// AlertsResponse is the structure of JSON response UI will use to get alert data
type AlertsResponse struct {
Status string `json:"status"`
Timestamp string `json:"timestamp"`
Version string `json:"version"`
Upstreams AlertmanagerAPISummary `json:"upstreams"`
AlertGroups map[string]APIAlertGroup `json:"groups"`
TotalAlerts int `json:"totalAlerts"`
Colors LabelsColorMap `json:"colors"`
Filters []Filter `json:"filters"`
Counters LabelsCountMap `json:"counters"`
StaticColorLabels []string `json:"staticColorLabels"`
Status string `json:"status"`
Timestamp string `json:"timestamp"`
Version string `json:"version"`
Upstreams AlertmanagerAPISummary `json:"upstreams"`
Silences map[string]map[string]Silence `json:"silences"`
AlertGroups map[string]APIAlertGroup `json:"groups"`
TotalAlerts int `json:"totalAlerts"`
Colors LabelsColorMap `json:"colors"`
Filters []Filter `json:"filters"`
Counters LabelsCountMap `json:"counters"`
StaticColorLabels []string `json:"staticColorLabels"`
}
// Autocomplete is the structure of autocomplete object for filter hints
views.go
+15
View File
@@ -76,6 +76,11 @@ func alerts(c *gin.Context) {
dedupedAlerts := alertmanager.DedupAlerts()
dedupedColors := alertmanager.DedupColors()
silences := map[string]map[string]models.Silence{}
for _, am := range alertmanager.GetAlertmanagers() {
silences[am.Name] = map[string]models.Silence{}
}
var matches int
for _, ag := range dedupedAlerts {
agCopy := models.AlertGroup{
@@ -137,6 +142,15 @@ func alerts(c *gin.Context) {
}
if len(agCopy.Alerts) > 0 {
for _, alert := range agCopy.Alerts {
if alert.IsSilenced() {
for _, am := range alert.Alertmanager {
for _, silence := range am.Silences {
silences[am.Name][silence.ID] = *silence
}
}
}
}
agCopy.Hash = agCopy.ContentFingerprint()
apiAG := models.APIAlertGroup{AlertGroup: agCopy}
apiAG.DedupSharedMaps()
@@ -147,6 +161,7 @@ func alerts(c *gin.Context) {
}
resp.AlertGroups = alerts
resp.Silences = silences
resp.Colors = colors
resp.Counters = counters