github/karma
1
0
Fork 0
mirror of https://github.com/prymitive/karma synced 2026-08-23 11:56:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #278 from prymitive/ha

Browse Source 
fix(ui): don't show @alertmanager labels on HA setup
This commit is contained in:
Łukasz Mierzwa
2018-11-30 20:14:06 +00:00
committed by GitHub
parent b699dc5049 4babccbf0e
commit d2d1bce747
20 changed files with 233 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
-1
View File
@@ -1,6 +1,5 @@
.build
.coverage
.tests
bindata_assetfs.go
karma
ui/build
.gitignore
+1
View File
@@ -8,3 +8,4 @@ ui/build
ui/coverage
ui/node_modules
vendor
TODO.md
Makefile
+6
View File
@@ -102,6 +102,12 @@ run-docker: docker-image
-p $(PORT):$(PORT) \
$(NAME):$(VERSION)
.PHONY: run-demo
run-demo:
docker build --build-arg VERSION=$(VERSION) -t $(NAME):demo -f demo/Dockerfile .
@docker rm -f $(NAME)-demo || true
docker run --name $(NAME)-demo -p $(PORT):$(PORT) -p 9093:9093 -p 9094:9094 $(NAME):demo
.PHONY: lint-git-ci
lint-git-ci: .build/deps-build-node.ok
ui/node_modules/.bin/commitlint-travis
alerts.go
+21 -4
View File
@@ -1,6 +1,9 @@
package main
import (
"sort"
"strings"
"github.com/prymitive/karma/internal/alertmanager"
"github.com/prymitive/karma/internal/filters"
"github.com/prymitive/karma/internal/models"
@@ -33,13 +36,23 @@ func countLabel(countStore models.LabelsCountMap, key string, val string) {
func getUpstreams() models.AlertmanagerAPISummary {
summary := models.AlertmanagerAPISummary{}
clusters := map[string][]string{}
upstreams := alertmanager.GetAlertmanagers()
for _, upstream := range upstreams {
members := upstream.ClusterMemberNames()
sort.Strings(members)
key := strings.Join(members[:], "\n")
if _, found := clusters[key]; !found {
clusters[key] = members
}
u := models.AlertmanagerAPIStatus{
Name: upstream.Name,
URI: upstream.SanitizedURI(),
PublicURI: upstream.PublicURI(),
Error: upstream.Error(),
Name: upstream.Name,
URI: upstream.SanitizedURI(),
PublicURI: upstream.PublicURI(),
Error: upstream.Error(),
Version: upstream.Version(),
ClusterMembers: members,
}
summary.Instances = append(summary.Instances, u)
@@ -51,5 +64,9 @@ func getUpstreams() models.AlertmanagerAPISummary {
}
}
for _, cluster := range clusters {
summary.Clusters = append(summary.Clusters, cluster)
}
return summary
}
demo/generator.py
+10 -6
View File
@@ -11,7 +11,10 @@ $ docker run \
-v $(pwd)/alertmanager.yaml:/etc/alertmanager/alertmanager.yml \
prom/alertmanager
2. Start this script
2. Start this script:
$ ./generator.py
3. Start karma:
$ karma \
@@ -24,14 +27,14 @@ $ karma \
"""
import random
import json
import datetime
import json
import random
import time
import urllib2
API = "http://localhost:9093"
APIs = ["http://localhost:9093", "http://localhost:9094"]
MAX_INTERVAL = 10
MIN_INTERVAL = 5
@@ -48,7 +51,7 @@ def jsonPostRequest(uri, data):
def addSilence(matchers, startsAt, endsAt, createdBy, comment):
uri = "{}/api/v1/silences".format(API)
uri = "{}/api/v1/silences".format(APIs[0])
silences = jsonGetRequest(uri)
found = False
@@ -72,7 +75,8 @@ def addSilence(matchers, startsAt, endsAt, createdBy, comment):
def addAlerts(alerts):
jsonPostRequest("{}/api/v1/alerts".format(API), alerts)
for api in APIs:
jsonPostRequest("{}/api/v1/alerts".format(api), alerts)
def newMatcher(name, value, isRegex):
demo/karma.yaml
+5 -1
View File
@@ -1,10 +1,14 @@
alertmanager:
interval: 10s
servers:
- name: demo
- name: ha1
uri: "http://localhost:9093"
timeout: 10s
proxy: true
- name: ha2
uri: "http://localhost:9094"
timeout: 10s
proxy: true
annotations:
hidden:
- help
demo/supervisord.conf
+10 -2
View File
@@ -6,8 +6,16 @@ logfile_maxbytes = 1MB
logfile_backups=0
loglevel = info
[program:alertmanager]
command=/alertmanager --config.file=/etc/alertmanager.yaml --storage.path=/tmp/alertmanager
[program:alertmanager1]
command=/alertmanager --config.file=/etc/alertmanager.yaml --storage.path=/tmp/alertmanager1 --web.listen-address=:9093 --cluster.listen-address=127.0.0.1:8001 --cluster.peer=127.0.0.1:8002
autorestart=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
[program:alertmanager2]
command=/alertmanager --config.file=/etc/alertmanager.yaml --storage.path=/tmp/alertmanager1 --web.listen-address=:9094 --cluster.listen-address=127.0.0.1:8002 --cluster.peer=127.0.0.1:8001
autorestart=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
internal/alertmanager/models.go
+100 -18
View File
@@ -14,6 +14,7 @@ import (
"github.com/prymitive/karma/internal/filters"
"github.com/prymitive/karma/internal/mapper"
"github.com/prymitive/karma/internal/models"
"github.com/prymitive/karma/internal/slices"
"github.com/prymitive/karma/internal/transform"
"github.com/prymitive/karma/internal/uri"
@@ -30,6 +31,12 @@ type alertmanagerMetrics struct {
errors map[string]float64
}
type alertmanagerStatus struct {
version string
amID string
peerIDs []string
}
// Alertmanager represents Alertmanager upstream instance
type Alertmanager struct {
URI string `json:"uri"`
@@ -51,49 +58,68 @@ type Alertmanager struct {
autocomplete []models.Autocomplete
knownLabels []string
lastError string
status alertmanagerStatus
// metrics tracked per alertmanager instance
metrics alertmanagerMetrics
}
func (am *Alertmanager) detectVersion() string {
// if everything fails assume Alertmanager is at latest possible version
defaultVersion := "999.0.0"
func (am *Alertmanager) fetchStatus() alertmanagerStatus {
status := alertmanagerStatus{
// if everything fails assume Alertmanager is at latest possible version
version: "999.0.0",
amID: "",
peerIDs: []string{},
}
url, err := uri.JoinURL(am.URI, "api/v1/status")
if err != nil {
log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", am.SanitizedURI(), err)
return defaultVersion
return status
}
ver := alertmanagerVersion{}
resp := alertmanagerStatusResponse{}
// read raw body from the source
source, err := am.reader.Read(url)
if err != nil {
log.Errorf("[%s] %s request failed: %s", am.Name, uri.SanitizeURI(url), err)
return defaultVersion
return status
}
defer source.Close()
// decode body as JSON
err = json.NewDecoder(source).Decode(&ver)
err = json.NewDecoder(source).Decode(&resp)
if err != nil {
log.Errorf("[%s] %s failed to decode as JSON: %s", am.Name, uri.SanitizeURI(url), err)
return defaultVersion
return status
}
if ver.Status != "success" {
log.Errorf("[%s] Request to %s returned status %s", am.Name, uri.SanitizeURI(url), ver.Status)
return defaultVersion
if resp.Status != "success" {
log.Errorf("[%s] Request to %s returned status %s", am.Name, uri.SanitizeURI(url), resp.Status)
return status
}
if ver.Data.VersionInfo.Version == "" {
if resp.Data.VersionInfo.Version == "" {
log.Errorf("[%s] No version information in Alertmanager API at %s", am.Name, uri.SanitizeURI(url))
return defaultVersion
return status
}
log.Infof("[%s] Remote Alertmanager version: %s", am.Name, ver.Data.VersionInfo.Version)
return ver.Data.VersionInfo.Version
status.version = resp.Data.VersionInfo.Version
log.Infof("[%s] Remote Alertmanager version: %s", am.Name, status.version)
if resp.Data.ClusterStatus.Name != "" {
status.amID = resp.Data.ClusterStatus.Name
for _, peer := range resp.Data.ClusterStatus.Peers {
status.peerIDs = append(status.peerIDs, peer.Name)
}
} else if resp.Data.MeshStatus.Name != "" {
status.amID = resp.Data.MeshStatus.Name
for _, peer := range resp.Data.MeshStatus.Peers {
status.peerIDs = append(status.peerIDs, peer.Name)
}
}
return status
}
func (am *Alertmanager) clearData() {
@@ -103,6 +129,11 @@ func (am *Alertmanager) clearData() {
am.colors = models.LabelsColorMap{}
am.autocomplete = []models.Autocomplete{}
am.knownLabels = []string{}
am.status = alertmanagerStatus{
version: "",
amID: "",
peerIDs: []string{},
}
am.lock.Unlock()
}
@@ -309,9 +340,9 @@ func (am *Alertmanager) pullAlerts(version string) error {
func (am *Alertmanager) Pull() error {
am.metrics.cycles++
version := am.detectVersion()
status := am.fetchStatus()
err := am.pullSilences(version)
err := am.pullSilences(status.version)
if err != nil {
am.clearData()
am.setError(err.Error())
@@ -319,7 +350,7 @@ func (am *Alertmanager) Pull() error {
return err
}
err = am.pullAlerts(version)
err = am.pullAlerts(status.version)
if err != nil {
am.clearData()
am.setError(err.Error())
@@ -327,7 +358,11 @@ func (am *Alertmanager) Pull() error {
return err
}
am.lock.Lock()
am.status = status
am.lastError = ""
am.lock.Unlock()
return nil
}
@@ -418,5 +453,52 @@ func (am *Alertmanager) Error() string {
// SanitizedURI returns a copy of Alertmanager.URI with password replaced by
// "xxx"
func (am *Alertmanager) SanitizedURI() string {
am.lock.RLock()
defer am.lock.RUnlock()
return uri.SanitizeURI(am.URI)
}
// Version returns last known version of this Alertmanager instance
func (am *Alertmanager) Version() string {
am.lock.RLock()
defer am.lock.RUnlock()
return am.status.version
}
// ClusterPeers returns a list of IDs of all peers this instance
// is connected to.
// IDs are the same as in Alertmanager API.
func (am *Alertmanager) ClusterPeers() []string {
am.lock.RLock()
defer am.lock.RUnlock()
return am.status.peerIDs
}
// ClusterMemberNames returns a list of names of all Alertmanager instances
// that are in the same cluster as this instance (including self).
// Names are the same as in karma configuration.
func (am *Alertmanager) ClusterMemberNames() []string {
am.lock.RLock()
defer am.lock.RUnlock()
members := []string{am.Name}
upstreams := GetAlertmanagers()
for _, upstream := range upstreams {
if upstream.Name == am.Name {
continue
}
for _, peerID := range upstream.ClusterPeers() {
if slices.StringInSlice(am.status.peerIDs, peerID) {
if !slices.StringInSlice(members, upstream.Name) {
members = append(members, upstream.Name)
}
}
}
}
return members
}
internal/alertmanager/status.go
+34
View File
@@ -0,0 +1,34 @@
package alertmanager
type v06MeshPeer struct {
Name string `json:"name"`
NickName string `json:"nickName"`
}
type v06CMeshStatus struct {
Name string `json:"name"`
NickName string `json:"nickName"`
Peers []v06MeshPeer `json:"peers"`
}
type v015ClusterPeer struct {
Address string `json:"address"`
Name string `json:"name"`
}
type v015ClusterStatus struct {
Name string `json:"name"`
Peers []v015ClusterPeer `json:"peers"`
Status string `json:"status"`
}
type alertmanagerStatusResponse struct {
Status string `json:"status"`
Data struct {
VersionInfo struct {
Version string `json:"version"`
} `json:"versionInfo"`
MeshStatus v06CMeshStatus `json:"meshStatus"`
ClusterStatus v015ClusterStatus `json:"clusterStatus"`
} `json:"data"`
}
internal/alertmanager/upstream.go
+1
View File
@@ -37,6 +37,7 @@ func NewAlertmanager(name, upstreamURI string, opts ...Option) (*Alertmanager, e
labelValueErrorsSilences: 0,
},
},
status: alertmanagerStatus{},
}
for _, opt := range opts {
internal/alertmanager/version.go
-12
View File
@@ -1,12 +0,0 @@
package alertmanager
// AlertmanagerVersion is what api/v1/status returns, we only use it to check
// version, so we skip all other keys (except for status)
type alertmanagerVersion struct {
Status string `json:"status"`
Data struct {
VersionInfo struct {
Version string `json:"version"`
} `json:"versionInfo"`
} `json:"data"`
}
internal/models/alertmanager.go
+5 -2
View File
@@ -29,8 +29,10 @@ type AlertmanagerAPIStatus struct {
URI string `json:"uri"`
// this is URI client should use to talk to this Alertmanager, it might be
// same as real or proxied URI
PublicURI string `json:"publicURI"`
Error string `json:"error"`
PublicURI string `json:"publicURI"`
Error string `json:"error"`
Version string `json:"version"`
ClusterMembers []string `json:"clusterMembers"`
}
// AlertmanagerAPICounters returns number of Alertmanager instances in each
@@ -45,4 +47,5 @@ type AlertmanagerAPICounters struct {
type AlertmanagerAPISummary struct {
Counters AlertmanagerAPICounters `json:"counters"`
Instances []AlertmanagerAPIStatus `json:"instances"`
Clusters [][]string `json:"clusters"`
}
ui/src/Components/Grid/AlertGrid/AlertGroup/Silence/index.test.js
+8 -3
View File
@@ -64,9 +64,12 @@ beforeEach(() => {
name: "default",
uri: "file:///mock",
publicURI: "http://example.com",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["default"]
}
]
],
clusters: [["default"]]
};
alertStore.data.silences = {
default: {
@@ -186,7 +189,9 @@ describe("<Silence />", () => {
name: "default",
uri: "file:///mock",
publicURI: "http://example.com",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["default"]
});
});
ui/src/Components/Grid/AlertGrid/index.js
+1 -1
View File
@@ -102,7 +102,7 @@ const AlertGrid = observer(
key={id}
group={alertStore.data.groups[id]}
showAlertmanagers={
alertStore.data.upstreams.instances.length > 1
alertStore.data.upstreams.clusters.length > 1
}
afterUpdate={this.masonryRepack}
settingsStore={settingsStore}
ui/src/Components/Grid/AlertGrid/index.test.js
+2 -1
View File
@@ -58,7 +58,8 @@ const MockGroupList = count => {
}
alertStore.data.upstreams = {
counters: { total: 0, healthy: 1, failed: 0 },
instances: [{ name: "am", uri: "http://am", error: "" }]
instances: [{ name: "am", uri: "http://am", error: "" }],
clusters: [["am"]]
};
alertStore.data.groups = groups;
};
ui/src/Components/Grid/index.test.js
+8 -4
View File
@@ -36,7 +36,8 @@ describe("<Grid />", () => {
it("renders FatalError if there's only one upstream and it's unhealthy", () => {
alertStore.data.upstreams = {
counters: { total: 1, healthy: 0, failed: 1 },
instances: [{ name: "am1", uri: "http://am1", error: "error" }]
instances: [{ name: "am1", uri: "http://am1", error: "error" }],
clusters: [["am1"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("<FatalError />");
@@ -45,7 +46,8 @@ describe("<Grid />", () => {
it("renders FatalError if there's only one upstream and it's unhealthy but without any error", () => {
alertStore.data.upstreams = {
counters: { total: 1, healthy: 0, failed: 1 },
instances: [{ name: "am1", uri: "http://am1", error: "" }]
instances: [{ name: "am1", uri: "http://am1", error: "" }],
clusters: [["am1"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("<AlertGrid />");
@@ -58,7 +60,8 @@ describe("<Grid />", () => {
{ name: "am1", uri: "http://am1", error: "error 1" },
{ name: "am2", uri: "file:///mock", error: "" },
{ name: "am3", uri: "http://am1", error: "error 2" }
]
],
clusters: [["am1"], ["am2"], ["am3"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("<UpstreamError /><UpstreamError /><AlertGrid />");
@@ -68,7 +71,8 @@ describe("<Grid />", () => {
alertStore.status.error = "error";
alertStore.data.upstreams = {
counters: { total: 0, healthy: 0, failed: 1 },
instances: [{ name: "am", uri: "http://am1", error: "error" }]
instances: [{ name: "am", uri: "http://am1", error: "error" }],
clusters: [["am"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("<FatalError />");
ui/src/Components/SilenceModal/AlertManagerInput/index.test.js
+12 -4
View File
@@ -23,19 +23,25 @@ beforeEach(() => {
name: "am1",
uri: "http://am1.example.com",
publicURI: "http://am1.example.com",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["am1"]
},
{
name: "am2",
uri: "http://am2.example.com",
publicURI: "http://am2.example.com",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["am2"]
},
{
name: "am3",
uri: "http://am3.example.com",
publicURI: "http://am3.example.com",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["am3"]
}
];
silenceFormStore = new SilenceFormStore();
@@ -139,7 +145,9 @@ describe("<AlertManagerInput />", () => {
alertStore.data.upstreams.instances[0] = {
name: "am1",
publicURI: "http://am1.example.com/new",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["am1"]
};
// force update since this is where the mismatch check lives
tree.instance().componentDidUpdate();
ui/src/Components/SilenceModal/SilenceSubmit/SilenceSubmitProgress.test.js
+3 -1
View File
@@ -17,7 +17,9 @@ beforeEach(() => {
name: "mockAlertmanager",
uri: "file:///mock",
publicURI: "http://example.com",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["mockAlertmanager"]
}
]
};
ui/src/Models/API.js
+3 -1
View File
@@ -68,7 +68,9 @@ const APIAlertmanagerUpstream = PropTypes.exact({
name: PropTypes.string.isRequired,
uri: PropTypes.string.isRequired,
publicURI: PropTypes.string.isRequired,
error: PropTypes.string.isRequired
error: PropTypes.string.isRequired,
version: PropTypes.string.isRequired,
clusterMembers: PropTypes.arrayOf(PropTypes.string).isRequired
});
export {
ui/src/__mocks__/Alerts.js
+3 -1
View File
@@ -68,7 +68,9 @@ const MockAlertmanager = () => ({
name: "default",
uri: "http://localhost",
publicURI: "http://am.example.com",
error: ""
error: "",
version: "0.15.0",
clusterMembers: ["default"]
});
export {