diff --git a/.dockerignore b/.dockerignore
index 6ed2e661b..c8de8bade 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,6 +1,5 @@
.build
.coverage
-.tests
bindata_assetfs.go
karma
ui/build
diff --git a/.gitignore b/.gitignore
index 7e12ba673..1bd92cca6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ ui/build
ui/coverage
ui/node_modules
vendor
+TODO.md
diff --git a/Makefile b/Makefile
index 222378de8..0337e46cb 100644
--- a/Makefile
+++ b/Makefile
@@ -102,6 +102,12 @@ run-docker: docker-image
-p $(PORT):$(PORT) \
$(NAME):$(VERSION)
+.PHONY: run-demo
+run-demo:
+ docker build --build-arg VERSION=$(VERSION) -t $(NAME):demo -f demo/Dockerfile .
+ @docker rm -f $(NAME)-demo || true
+ docker run --name $(NAME)-demo -p $(PORT):$(PORT) -p 9093:9093 -p 9094:9094 $(NAME):demo
+
.PHONY: lint-git-ci
lint-git-ci: .build/deps-build-node.ok
ui/node_modules/.bin/commitlint-travis
diff --git a/alerts.go b/alerts.go
index b8da47b76..267c7f189 100644
--- a/alerts.go
+++ b/alerts.go
@@ -1,6 +1,9 @@
package main
import (
+ "sort"
+ "strings"
+
"github.com/prymitive/karma/internal/alertmanager"
"github.com/prymitive/karma/internal/filters"
"github.com/prymitive/karma/internal/models"
@@ -33,13 +36,23 @@ func countLabel(countStore models.LabelsCountMap, key string, val string) {
func getUpstreams() models.AlertmanagerAPISummary {
summary := models.AlertmanagerAPISummary{}
+ clusters := map[string][]string{}
upstreams := alertmanager.GetAlertmanagers()
for _, upstream := range upstreams {
+ members := upstream.ClusterMemberNames()
+ sort.Strings(members)
+ key := strings.Join(members[:], "\n")
+ if _, found := clusters[key]; !found {
+ clusters[key] = members
+ }
+
u := models.AlertmanagerAPIStatus{
- Name: upstream.Name,
- URI: upstream.SanitizedURI(),
- PublicURI: upstream.PublicURI(),
- Error: upstream.Error(),
+ Name: upstream.Name,
+ URI: upstream.SanitizedURI(),
+ PublicURI: upstream.PublicURI(),
+ Error: upstream.Error(),
+ Version: upstream.Version(),
+ ClusterMembers: members,
}
summary.Instances = append(summary.Instances, u)
@@ -51,5 +64,9 @@ func getUpstreams() models.AlertmanagerAPISummary {
}
}
+ for _, cluster := range clusters {
+ summary.Clusters = append(summary.Clusters, cluster)
+ }
+
return summary
}
diff --git a/demo/generator.py b/demo/generator.py
index 9e31614a1..35f11c359 100755
--- a/demo/generator.py
+++ b/demo/generator.py
@@ -11,7 +11,10 @@ $ docker run \
-v $(pwd)/alertmanager.yaml:/etc/alertmanager/alertmanager.yml \
prom/alertmanager
-2. Start this script
+2. Start this script:
+
+$ ./generator.py
+
3. Start karma:
$ karma \
@@ -24,14 +27,14 @@ $ karma \
"""
-import random
-import json
import datetime
+import json
+import random
import time
import urllib2
-API = "http://localhost:9093"
+APIs = ["http://localhost:9093", "http://localhost:9094"]
MAX_INTERVAL = 10
MIN_INTERVAL = 5
@@ -48,7 +51,7 @@ def jsonPostRequest(uri, data):
def addSilence(matchers, startsAt, endsAt, createdBy, comment):
- uri = "{}/api/v1/silences".format(API)
+ uri = "{}/api/v1/silences".format(APIs[0])
silences = jsonGetRequest(uri)
found = False
@@ -72,7 +75,8 @@ def addSilence(matchers, startsAt, endsAt, createdBy, comment):
def addAlerts(alerts):
- jsonPostRequest("{}/api/v1/alerts".format(API), alerts)
+ for api in APIs:
+ jsonPostRequest("{}/api/v1/alerts".format(api), alerts)
def newMatcher(name, value, isRegex):
diff --git a/demo/karma.yaml b/demo/karma.yaml
index 145db0d96..8f9f1e2ee 100644
--- a/demo/karma.yaml
+++ b/demo/karma.yaml
@@ -1,10 +1,14 @@
alertmanager:
interval: 10s
servers:
- - name: demo
+ - name: ha1
uri: "http://localhost:9093"
timeout: 10s
proxy: true
+ - name: ha2
+ uri: "http://localhost:9094"
+ timeout: 10s
+ proxy: true
annotations:
hidden:
- help
diff --git a/demo/supervisord.conf b/demo/supervisord.conf
index 61eb3e01d..3fd5aa19e 100644
--- a/demo/supervisord.conf
+++ b/demo/supervisord.conf
@@ -6,8 +6,16 @@ logfile_maxbytes = 1MB
logfile_backups=0
loglevel = info
-[program:alertmanager]
-command=/alertmanager --config.file=/etc/alertmanager.yaml --storage.path=/tmp/alertmanager
+[program:alertmanager1]
+command=/alertmanager --config.file=/etc/alertmanager.yaml --storage.path=/tmp/alertmanager1 --web.listen-address=:9093 --cluster.listen-address=127.0.0.1:8001 --cluster.peer=127.0.0.1:8002
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:alertmanager2]
+command=/alertmanager --config.file=/etc/alertmanager.yaml --storage.path=/tmp/alertmanager1 --web.listen-address=:9094 --cluster.listen-address=127.0.0.1:8002 --cluster.peer=127.0.0.1:8001
autorestart=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
diff --git a/internal/alertmanager/models.go b/internal/alertmanager/models.go
index 21f5d8a1d..3cc9081c1 100644
--- a/internal/alertmanager/models.go
+++ b/internal/alertmanager/models.go
@@ -14,6 +14,7 @@ import (
"github.com/prymitive/karma/internal/filters"
"github.com/prymitive/karma/internal/mapper"
"github.com/prymitive/karma/internal/models"
+ "github.com/prymitive/karma/internal/slices"
"github.com/prymitive/karma/internal/transform"
"github.com/prymitive/karma/internal/uri"
@@ -30,6 +31,12 @@ type alertmanagerMetrics struct {
errors map[string]float64
}
+type alertmanagerStatus struct {
+ version string
+ amID string
+ peerIDs []string
+}
+
// Alertmanager represents Alertmanager upstream instance
type Alertmanager struct {
URI string `json:"uri"`
@@ -51,49 +58,68 @@ type Alertmanager struct {
autocomplete []models.Autocomplete
knownLabels []string
lastError string
+ status alertmanagerStatus
// metrics tracked per alertmanager instance
metrics alertmanagerMetrics
}
-func (am *Alertmanager) detectVersion() string {
- // if everything fails assume Alertmanager is at latest possible version
- defaultVersion := "999.0.0"
+func (am *Alertmanager) fetchStatus() alertmanagerStatus {
+ status := alertmanagerStatus{
+ // if everything fails assume Alertmanager is at latest possible version
+ version: "999.0.0",
+ amID: "",
+ peerIDs: []string{},
+ }
url, err := uri.JoinURL(am.URI, "api/v1/status")
if err != nil {
log.Errorf("Failed to join url '%s' and path 'api/v1/status': %s", am.SanitizedURI(), err)
- return defaultVersion
+ return status
}
- ver := alertmanagerVersion{}
+ resp := alertmanagerStatusResponse{}
// read raw body from the source
source, err := am.reader.Read(url)
if err != nil {
log.Errorf("[%s] %s request failed: %s", am.Name, uri.SanitizeURI(url), err)
- return defaultVersion
+ return status
}
defer source.Close()
// decode body as JSON
- err = json.NewDecoder(source).Decode(&ver)
+ err = json.NewDecoder(source).Decode(&resp)
if err != nil {
log.Errorf("[%s] %s failed to decode as JSON: %s", am.Name, uri.SanitizeURI(url), err)
- return defaultVersion
+ return status
}
- if ver.Status != "success" {
- log.Errorf("[%s] Request to %s returned status %s", am.Name, uri.SanitizeURI(url), ver.Status)
- return defaultVersion
+ if resp.Status != "success" {
+ log.Errorf("[%s] Request to %s returned status %s", am.Name, uri.SanitizeURI(url), resp.Status)
+ return status
}
- if ver.Data.VersionInfo.Version == "" {
+ if resp.Data.VersionInfo.Version == "" {
log.Errorf("[%s] No version information in Alertmanager API at %s", am.Name, uri.SanitizeURI(url))
- return defaultVersion
+ return status
}
- log.Infof("[%s] Remote Alertmanager version: %s", am.Name, ver.Data.VersionInfo.Version)
- return ver.Data.VersionInfo.Version
+ status.version = resp.Data.VersionInfo.Version
+ log.Infof("[%s] Remote Alertmanager version: %s", am.Name, status.version)
+
+ if resp.Data.ClusterStatus.Name != "" {
+ status.amID = resp.Data.ClusterStatus.Name
+ for _, peer := range resp.Data.ClusterStatus.Peers {
+ status.peerIDs = append(status.peerIDs, peer.Name)
+ }
+ } else if resp.Data.MeshStatus.Name != "" {
+ status.amID = resp.Data.MeshStatus.Name
+ for _, peer := range resp.Data.MeshStatus.Peers {
+ status.peerIDs = append(status.peerIDs, peer.Name)
+ }
+ }
+
+ return status
}
func (am *Alertmanager) clearData() {
@@ -103,6 +129,11 @@ func (am *Alertmanager) clearData() {
am.colors = models.LabelsColorMap{}
am.autocomplete = []models.Autocomplete{}
am.knownLabels = []string{}
+ am.status = alertmanagerStatus{
+ version: "",
+ amID: "",
+ peerIDs: []string{},
+ }
am.lock.Unlock()
}
@@ -309,9 +340,9 @@ func (am *Alertmanager) pullAlerts(version string) error {
func (am *Alertmanager) Pull() error {
am.metrics.cycles++
- version := am.detectVersion()
+ status := am.fetchStatus()
- err := am.pullSilences(version)
+ err := am.pullSilences(status.version)
if err != nil {
am.clearData()
am.setError(err.Error())
@@ -319,7 +350,7 @@ func (am *Alertmanager) Pull() error {
return err
}
- err = am.pullAlerts(version)
+ err = am.pullAlerts(status.version)
if err != nil {
am.clearData()
am.setError(err.Error())
@@ -327,7 +358,11 @@ func (am *Alertmanager) Pull() error {
return err
}
+ am.lock.Lock()
+ am.status = status
am.lastError = ""
+ am.lock.Unlock()
+
return nil
}
@@ -418,5 +453,52 @@ func (am *Alertmanager) Error() string {
// SanitizedURI returns a copy of Alertmanager.URI with password replaced by
// "xxx"
func (am *Alertmanager) SanitizedURI() string {
+ am.lock.RLock()
+ defer am.lock.RUnlock()
+
return uri.SanitizeURI(am.URI)
}
+
+// Version returns last known version of this Alertmanager instance
+func (am *Alertmanager) Version() string {
+ am.lock.RLock()
+ defer am.lock.RUnlock()
+
+ return am.status.version
+}
+
+// ClusterPeers returns a list of IDs of all peers this instance
+// is connected to.
+// IDs are the same as in Alertmanager API.
+func (am *Alertmanager) ClusterPeers() []string {
+ am.lock.RLock()
+ defer am.lock.RUnlock()
+
+ return am.status.peerIDs
+}
+
+// ClusterMemberNames returns a list of names of all Alertmanager instances
+// that are in the same cluster as this instance (including self).
+// Names are the same as in karma configuration.
+func (am *Alertmanager) ClusterMemberNames() []string {
+ am.lock.RLock()
+ defer am.lock.RUnlock()
+
+ members := []string{am.Name}
+
+ upstreams := GetAlertmanagers()
+ for _, upstream := range upstreams {
+ if upstream.Name == am.Name {
+ continue
+ }
+ for _, peerID := range upstream.ClusterPeers() {
+ if slices.StringInSlice(am.status.peerIDs, peerID) {
+ if !slices.StringInSlice(members, upstream.Name) {
+ members = append(members, upstream.Name)
+ }
+ }
+ }
+ }
+
+ return members
+}
diff --git a/internal/alertmanager/status.go b/internal/alertmanager/status.go
new file mode 100644
index 000000000..0ee0a5c2d
--- /dev/null
+++ b/internal/alertmanager/status.go
@@ -0,0 +1,34 @@
+package alertmanager
+
+type v06MeshPeer struct {
+ Name string `json:"name"`
+ NickName string `json:"nickName"`
+}
+
+type v06CMeshStatus struct {
+ Name string `json:"name"`
+ NickName string `json:"nickName"`
+ Peers []v06MeshPeer `json:"peers"`
+}
+
+type v015ClusterPeer struct {
+ Address string `json:"address"`
+ Name string `json:"name"`
+}
+
+type v015ClusterStatus struct {
+ Name string `json:"name"`
+ Peers []v015ClusterPeer `json:"peers"`
+ Status string `json:"status"`
+}
+
+type alertmanagerStatusResponse struct {
+ Status string `json:"status"`
+ Data struct {
+ VersionInfo struct {
+ Version string `json:"version"`
+ } `json:"versionInfo"`
+ MeshStatus v06CMeshStatus `json:"meshStatus"`
+ ClusterStatus v015ClusterStatus `json:"clusterStatus"`
+ } `json:"data"`
+}
diff --git a/internal/alertmanager/upstream.go b/internal/alertmanager/upstream.go
index c7e202b1b..01dff5254 100644
--- a/internal/alertmanager/upstream.go
+++ b/internal/alertmanager/upstream.go
@@ -37,6 +37,7 @@ func NewAlertmanager(name, upstreamURI string, opts ...Option) (*Alertmanager, e
labelValueErrorsSilences: 0,
},
},
+ status: alertmanagerStatus{},
}
for _, opt := range opts {
diff --git a/internal/alertmanager/version.go b/internal/alertmanager/version.go
deleted file mode 100644
index b001493e0..000000000
--- a/internal/alertmanager/version.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package alertmanager
-
-// AlertmanagerVersion is what api/v1/status returns, we only use it to check
-// version, so we skip all other keys (except for status)
-type alertmanagerVersion struct {
- Status string `json:"status"`
- Data struct {
- VersionInfo struct {
- Version string `json:"version"`
- } `json:"versionInfo"`
- } `json:"data"`
-}
diff --git a/internal/models/alertmanager.go b/internal/models/alertmanager.go
index 545160d95..718322822 100644
--- a/internal/models/alertmanager.go
+++ b/internal/models/alertmanager.go
@@ -29,8 +29,10 @@ type AlertmanagerAPIStatus struct {
URI string `json:"uri"`
// this is URI client should use to talk to this Alertmanager, it might be
// same as real or proxied URI
- PublicURI string `json:"publicURI"`
- Error string `json:"error"`
+ PublicURI string `json:"publicURI"`
+ Error string `json:"error"`
+ Version string `json:"version"`
+ ClusterMembers []string `json:"clusterMembers"`
}
// AlertmanagerAPICounters returns number of Alertmanager instances in each
@@ -45,4 +47,5 @@ type AlertmanagerAPICounters struct {
type AlertmanagerAPISummary struct {
Counters AlertmanagerAPICounters `json:"counters"`
Instances []AlertmanagerAPIStatus `json:"instances"`
+ Clusters [][]string `json:"clusters"`
}
diff --git a/ui/src/Components/Grid/AlertGrid/AlertGroup/Silence/index.test.js b/ui/src/Components/Grid/AlertGrid/AlertGroup/Silence/index.test.js
index d5276ab4f..293aa7d9e 100644
--- a/ui/src/Components/Grid/AlertGrid/AlertGroup/Silence/index.test.js
+++ b/ui/src/Components/Grid/AlertGrid/AlertGroup/Silence/index.test.js
@@ -64,9 +64,12 @@ beforeEach(() => {
name: "default",
uri: "file:///mock",
publicURI: "http://example.com",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["default"]
}
- ]
+ ],
+ clusters: [["default"]]
};
alertStore.data.silences = {
default: {
@@ -186,7 +189,9 @@ describe("", () => {
name: "default",
uri: "file:///mock",
publicURI: "http://example.com",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["default"]
});
});
diff --git a/ui/src/Components/Grid/AlertGrid/index.js b/ui/src/Components/Grid/AlertGrid/index.js
index fb671605a..51b80c94e 100644
--- a/ui/src/Components/Grid/AlertGrid/index.js
+++ b/ui/src/Components/Grid/AlertGrid/index.js
@@ -102,7 +102,7 @@ const AlertGrid = observer(
key={id}
group={alertStore.data.groups[id]}
showAlertmanagers={
- alertStore.data.upstreams.instances.length > 1
+ alertStore.data.upstreams.clusters.length > 1
}
afterUpdate={this.masonryRepack}
settingsStore={settingsStore}
diff --git a/ui/src/Components/Grid/AlertGrid/index.test.js b/ui/src/Components/Grid/AlertGrid/index.test.js
index c789f9ba6..2ce7457d7 100644
--- a/ui/src/Components/Grid/AlertGrid/index.test.js
+++ b/ui/src/Components/Grid/AlertGrid/index.test.js
@@ -58,7 +58,8 @@ const MockGroupList = count => {
}
alertStore.data.upstreams = {
counters: { total: 0, healthy: 1, failed: 0 },
- instances: [{ name: "am", uri: "http://am", error: "" }]
+ instances: [{ name: "am", uri: "http://am", error: "" }],
+ clusters: [["am"]]
};
alertStore.data.groups = groups;
};
diff --git a/ui/src/Components/Grid/index.test.js b/ui/src/Components/Grid/index.test.js
index 42e491a39..0ad096b26 100644
--- a/ui/src/Components/Grid/index.test.js
+++ b/ui/src/Components/Grid/index.test.js
@@ -36,7 +36,8 @@ describe("", () => {
it("renders FatalError if there's only one upstream and it's unhealthy", () => {
alertStore.data.upstreams = {
counters: { total: 1, healthy: 0, failed: 1 },
- instances: [{ name: "am1", uri: "http://am1", error: "error" }]
+ instances: [{ name: "am1", uri: "http://am1", error: "error" }],
+ clusters: [["am1"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("");
@@ -45,7 +46,8 @@ describe("", () => {
it("renders FatalError if there's only one upstream and it's unhealthy but without any error", () => {
alertStore.data.upstreams = {
counters: { total: 1, healthy: 0, failed: 1 },
- instances: [{ name: "am1", uri: "http://am1", error: "" }]
+ instances: [{ name: "am1", uri: "http://am1", error: "" }],
+ clusters: [["am1"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("");
@@ -58,7 +60,8 @@ describe("", () => {
{ name: "am1", uri: "http://am1", error: "error 1" },
{ name: "am2", uri: "file:///mock", error: "" },
{ name: "am3", uri: "http://am1", error: "error 2" }
- ]
+ ],
+ clusters: [["am1"], ["am2"], ["am3"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("");
@@ -68,7 +71,8 @@ describe("", () => {
alertStore.status.error = "error";
alertStore.data.upstreams = {
counters: { total: 0, healthy: 0, failed: 1 },
- instances: [{ name: "am", uri: "http://am1", error: "error" }]
+ instances: [{ name: "am", uri: "http://am1", error: "error" }],
+ clusters: [["am"]]
};
const tree = ShallowGrid();
expect(tree.text()).toBe("");
diff --git a/ui/src/Components/SilenceModal/AlertManagerInput/index.test.js b/ui/src/Components/SilenceModal/AlertManagerInput/index.test.js
index c11c6f25d..053baef9c 100644
--- a/ui/src/Components/SilenceModal/AlertManagerInput/index.test.js
+++ b/ui/src/Components/SilenceModal/AlertManagerInput/index.test.js
@@ -23,19 +23,25 @@ beforeEach(() => {
name: "am1",
uri: "http://am1.example.com",
publicURI: "http://am1.example.com",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["am1"]
},
{
name: "am2",
uri: "http://am2.example.com",
publicURI: "http://am2.example.com",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["am2"]
},
{
name: "am3",
uri: "http://am3.example.com",
publicURI: "http://am3.example.com",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["am3"]
}
];
silenceFormStore = new SilenceFormStore();
@@ -139,7 +145,9 @@ describe("", () => {
alertStore.data.upstreams.instances[0] = {
name: "am1",
publicURI: "http://am1.example.com/new",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["am1"]
};
// force update since this is where the mismatch check lives
tree.instance().componentDidUpdate();
diff --git a/ui/src/Components/SilenceModal/SilenceSubmit/SilenceSubmitProgress.test.js b/ui/src/Components/SilenceModal/SilenceSubmit/SilenceSubmitProgress.test.js
index 1f95f320a..426673055 100644
--- a/ui/src/Components/SilenceModal/SilenceSubmit/SilenceSubmitProgress.test.js
+++ b/ui/src/Components/SilenceModal/SilenceSubmit/SilenceSubmitProgress.test.js
@@ -17,7 +17,9 @@ beforeEach(() => {
name: "mockAlertmanager",
uri: "file:///mock",
publicURI: "http://example.com",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["mockAlertmanager"]
}
]
};
diff --git a/ui/src/Models/API.js b/ui/src/Models/API.js
index 3959c35f9..9f837e44f 100644
--- a/ui/src/Models/API.js
+++ b/ui/src/Models/API.js
@@ -68,7 +68,9 @@ const APIAlertmanagerUpstream = PropTypes.exact({
name: PropTypes.string.isRequired,
uri: PropTypes.string.isRequired,
publicURI: PropTypes.string.isRequired,
- error: PropTypes.string.isRequired
+ error: PropTypes.string.isRequired,
+ version: PropTypes.string.isRequired,
+ clusterMembers: PropTypes.arrayOf(PropTypes.string).isRequired
});
export {
diff --git a/ui/src/__mocks__/Alerts.js b/ui/src/__mocks__/Alerts.js
index ae8a012de..19e8c5077 100644
--- a/ui/src/__mocks__/Alerts.js
+++ b/ui/src/__mocks__/Alerts.js
@@ -68,7 +68,9 @@ const MockAlertmanager = () => ({
name: "default",
uri: "http://localhost",
publicURI: "http://am.example.com",
- error: ""
+ error: "",
+ version: "0.15.0",
+ clusterMembers: ["default"]
});
export {