From b6dd993c2bfa39c81aaa8391286879ab9d3e7095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Wed, 6 Dec 2017 22:46:21 -0800 Subject: [PATCH 1/8] Add support for proxying user connection to Alertmanager Fixes #190. With this feature unsee can be configured to proxy requests to selected Alertmanager instances, if it's enabled unsee silence form will send a request via unsee rather than directly. This allows users to manage silences in environments where they have access to unsee but not to Alertmanager. Only silences endpoints on Alertmanager API are proxied. --- docs/CONFIGURATION.md | 10 ++- docs/example.yaml | 6 +- internal/alertmanager/dedup_test.go | 2 +- internal/alertmanager/models.go | 23 +++++- internal/alertmanager/upstream.go | 19 ++--- internal/config/config_test.go | 1 + internal/config/models.go | 1 + internal/filters/filter_test.go | 2 +- main.go | 5 +- proxy.go | 56 +++++++++++++ proxy_test.go | 121 ++++++++++++++++++++++++++++ 11 files changed, 230 insertions(+), 16 deletions(-) create mode 100644 proxy.go create mode 100644 proxy_test.go diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 30c9a45ed..da6912a00 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -49,6 +49,7 @@ alertmanager: - name: string uri: string timeout: duration + proxy: bool ``` * `interval` - how often alerts should be refreshed, a string in @@ -70,8 +71,12 @@ alertmanager: of unsee with `make run`. * `timeout` - timeout for requests send to this Alertmanager server, a string in [time.Duration](https://golang.org/pkg/time/#ParseDuration) format. +* `proxy` - if enabled requests from user browsers to this Alertmanager will be + proxied via unsee. This applies to requests made when managing + silences via unsee (creating or expiring silences). -Example: +Example with two production Alertmanager instances running in HA mode and a +staging instance that is also proxied: ```yaml alertmanager: @@ -80,12 +85,15 @@ alertmanager: - name: production1 uri: https://alertmanager1.prod.example.com timeout: 20s + proxy: false - name: production2 uri: https://alertmanager2.prod.example.com timeout: 20s + proxy: false - name: staging uri: https://alertmanager.staging.example.com timeout: 30s + proxy: true ``` Defaults: diff --git a/docs/example.yaml b/docs/example.yaml index 01b19dc45..043791adb 100644 --- a/docs/example.yaml +++ b/docs/example.yaml @@ -1,9 +1,10 @@ alertmanager: interval: 60s servers: - - name: mock - uri: file://internal/mock/0.11.0 + - name: local + uri: http://localhost:9093 timeout: 10s + proxy: true annotations: default: hidden: false @@ -29,6 +30,7 @@ listen: port: 8080 prefix: / log: + config: false level: info jira: - regex: DEVOPS-[0-9]+ diff --git a/internal/alertmanager/dedup_test.go b/internal/alertmanager/dedup_test.go index 039f9ec9e..f102660c2 100644 --- a/internal/alertmanager/dedup_test.go +++ b/internal/alertmanager/dedup_test.go @@ -16,7 +16,7 @@ import ( func init() { log.SetLevel(log.ErrorLevel) for i, uri := range mock.ListAllMockURIs() { - alertmanager.NewAlertmanager(fmt.Sprintf("dedup-mock-%d", i), uri, time.Second) + alertmanager.NewAlertmanager(fmt.Sprintf("dedup-mock-%d", i), uri, time.Second, false) } } diff --git a/internal/alertmanager/models.go b/internal/alertmanager/models.go index 344eb0e19..809ba0edc 100644 --- a/internal/alertmanager/models.go +++ b/internal/alertmanager/models.go @@ -2,10 +2,13 @@ package alertmanager import ( "fmt" + "path" "sort" + "strings" "sync" "time" + "github.com/cloudflare/unsee/internal/config" "github.com/cloudflare/unsee/internal/mapper" "github.com/cloudflare/unsee/internal/models" "github.com/cloudflare/unsee/internal/transform" @@ -29,6 +32,8 @@ type Alertmanager struct { URI string `json:"uri"` Timeout time.Duration `json:"timeout"` Name string `json:"name"` + // whenever this instance should be proxied + ProxyRequests bool // lock protects data access while updating lock sync.RWMutex // fields for storing pulled data @@ -107,6 +112,22 @@ func (am *Alertmanager) pullSilences(version string) error { return nil } +// this is the URI of this Alertmanager we put in JSON reponse +// it's either real full URI or a proxy relative URI +func (am *Alertmanager) publicURI() string { + if am.ProxyRequests { + sub := fmt.Sprintf("/proxy/alertmanager/%s", am.Name) + uri := path.Join(config.Config.Listen.Prefix, sub) + if strings.HasSuffix(sub, "/") { + // if sub path had trailing slash then add it here, since path.Join will + // skip it + return uri + "/" + } + return uri + } + return am.URI +} + func (am *Alertmanager) pullAlerts(version string) error { mapper, err := mapper.GetAlertMapper(version) if err != nil { @@ -163,7 +184,7 @@ func (am *Alertmanager) pullAlerts(version string) error { alert.Alertmanager = []models.AlertmanagerInstance{ models.AlertmanagerInstance{ Name: am.Name, - URI: am.URI, + URI: am.publicURI(), State: alert.State, StartsAt: alert.StartsAt, EndsAt: alert.EndsAt, diff --git a/internal/alertmanager/upstream.go b/internal/alertmanager/upstream.go index 0423d1d2d..e6b025772 100644 --- a/internal/alertmanager/upstream.go +++ b/internal/alertmanager/upstream.go @@ -15,7 +15,7 @@ var ( ) // NewAlertmanager creates a new Alertmanager instance -func NewAlertmanager(name, uri string, timeout time.Duration) error { +func NewAlertmanager(name, uri string, timeout time.Duration, proxyRequests bool) error { if _, found := upstreams[name]; found { return fmt.Errorf("Alertmanager upstream '%s' already exist", name) } @@ -27,14 +27,15 @@ func NewAlertmanager(name, uri string, timeout time.Duration) error { } upstreams[name] = &Alertmanager{ - URI: uri, - Timeout: timeout, - Name: name, - lock: sync.RWMutex{}, - alertGroups: []models.AlertGroup{}, - silences: map[string]models.Silence{}, - colors: models.LabelsColorMap{}, - autocomplete: []models.Autocomplete{}, + URI: uri, + Timeout: timeout, + Name: name, + ProxyRequests: proxyRequests, + lock: sync.RWMutex{}, + alertGroups: []models.AlertGroup{}, + silences: map[string]models.Silence{}, + colors: models.LabelsColorMap{}, + autocomplete: []models.Autocomplete{}, metrics: alertmanagerMetrics{ errors: map[string]float64{ labelValueErrorsAlerts: 0, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b0608c636..ffdafe7a9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -53,6 +53,7 @@ func testReadConfig(t *testing.T) { - name: default uri: http://localhost timeout: 40s + proxy: false annotations: default: hidden: true diff --git a/internal/config/models.go b/internal/config/models.go index aeaab158a..cee46e9bc 100644 --- a/internal/config/models.go +++ b/internal/config/models.go @@ -6,6 +6,7 @@ type alertmanagerConfig struct { Name string URI string Timeout time.Duration + Proxy bool } type jiraRule struct { diff --git a/internal/filters/filter_test.go b/internal/filters/filter_test.go index 9f39d8225..819608218 100644 --- a/internal/filters/filter_test.go +++ b/internal/filters/filter_test.go @@ -485,7 +485,7 @@ var tests = []filterTest{ func TestFilters(t *testing.T) { log.SetLevel(log.ErrorLevel) - err := alertmanager.NewAlertmanager("test", "http://localhost", time.Second) + err := alertmanager.NewAlertmanager("test", "http://localhost", time.Second, false) am := alertmanager.GetAlertmanagerByName("test") if err != nil { t.Error(err) diff --git a/main.go b/main.go index 20e31e5eb..056e57ffe 100644 --- a/main.go +++ b/main.go @@ -60,7 +60,7 @@ func setupRouter(router *gin.Engine) { func setupUpstreams() { for _, s := range config.Config.Alertmanager.Servers { - err := alertmanager.NewAlertmanager(s.Name, s.URI, s.Timeout) + err := alertmanager.NewAlertmanager(s.Name, s.URI, s.Timeout, s.Proxy) if err != nil { log.Fatalf("Failed to configure Alertmanager '%s' with URI '%s': %s", s.Name, s.URI, err) } @@ -151,6 +151,9 @@ func main() { } setupRouter(router) + for _, am := range alertmanager.GetAlertmanagers() { + setupRouterProxyHandlers(router, am) + } listen := fmt.Sprintf("%s:%d", config.Config.Listen.Address, config.Config.Listen.Port) log.Infof("Listening on %s", listen) err := router.Run(listen) diff --git a/proxy.go b/proxy.go new file mode 100644 index 000000000..bc56adf80 --- /dev/null +++ b/proxy.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + "github.com/cloudflare/unsee/internal/alertmanager" + "github.com/cloudflare/unsee/internal/config" + "github.com/gin-gonic/gin" + + log "github.com/sirupsen/logrus" +) + +func proxyPathPrefix(name string) string { + return fmt.Sprintf("%sproxy/alertmanager/%s", config.Config.Listen.Prefix, name) +} + +// NewAlertmanagerProxy creates a proxy instance for given alertmanager instance +func NewAlertmanagerProxy(alertmanager *alertmanager.Alertmanager) (*httputil.ReverseProxy, error) { + upstreamURL, err := url.Parse(alertmanager.URI) + if err != nil { + return nil, err + } + proxy := httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL.Scheme = upstreamURL.Scheme + req.URL.Host = upstreamURL.Host + req.URL.Path = strings.TrimPrefix(req.URL.Path, proxyPathPrefix(alertmanager.Name)) + // drop Accept-Encoding header so we always get uncompressed reponses from + // upstream, there's a gzip middleware that's global so we don't want it + // to gzip twice + req.Header.Del("Accept-Encoding") + log.Debugf("[%s] Proxy request for %s", alertmanager.Name, req.URL.Path) + }, + ModifyResponse: func(resp *http.Response) error { + // drop Content-Length header from upstream responses, gzip middleware + // will compress those and that could cause a mismatch + resp.Header.Del("Content-Length") + return nil + }, + } + return &proxy, nil +} + +func setupRouterProxyHandlers(router *gin.Engine, alertmanager *alertmanager.Alertmanager) error { + proxy, err := NewAlertmanagerProxy(alertmanager) + if err != nil { + return err + } + router.POST(fmt.Sprintf("%s/api/v1/silences", proxyPathPrefix(alertmanager.Name)), gin.WrapH(proxy)) + router.DELETE(fmt.Sprintf("%s/api/v1/silence/*id", proxyPathPrefix(alertmanager.Name)), gin.WrapH(proxy)) + return nil +} diff --git a/proxy_test.go b/proxy_test.go new file mode 100644 index 000000000..e40c97b0c --- /dev/null +++ b/proxy_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/cloudflare/unsee/internal/alertmanager" + + httpmock "gopkg.in/jarcoal/httpmock.v1" +) + +// httptest.NewRecorder() doesn't implement http.CloseNotifier +type closeNotifyingRecorder struct { + *httptest.ResponseRecorder + closed chan bool +} + +func newCloseNotifyingRecorder() *closeNotifyingRecorder { + return &closeNotifyingRecorder{ + httptest.NewRecorder(), + make(chan bool, 1), + } +} + +func (c *closeNotifyingRecorder) close() { + c.closed <- true +} + +func (c *closeNotifyingRecorder) CloseNotify() <-chan bool { + return c.closed +} + +type proxyTest struct { + method string + localPath string + upstreamURI string + code int + response string +} + +var proxyTests = []proxyTest{ + // valid alertmanager and methods + proxyTest{ + method: "POST", + localPath: "/proxy/alertmanager/dummy/api/v1/silences", + upstreamURI: "http://localhost:9093/api/v1/silences", + code: 200, + response: "{\"status\":\"success\",\"data\":{\"silenceId\":\"d8a61ca8-ee2e-4076-999f-276f1e986bf3\"}}", + }, + proxyTest{ + method: "DELETE", + localPath: "/proxy/alertmanager/dummy/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", + upstreamURI: "http://localhost:9093/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", + code: 200, + response: "{\"status\":\"success\"}", + }, + // invalid alertmanager name + proxyTest{ + method: "POST", + localPath: "/proxy/alertmanager/INVALID/api/v1/silences", + upstreamURI: "", + code: 404, + response: "404 page not found", + }, + proxyTest{ + method: "DELETE", + localPath: "/proxy/alertmanager/INVALID/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", + upstreamURI: "http://localhost:9093/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", + code: 404, + response: "404 page not found", + }, + // valid alertmanager name, but invalid method + proxyTest{ + method: "GET", + localPath: "/proxy/alertmanager/dummy/api/v1/silences", + upstreamURI: "", + code: 404, + response: "404 page not found", + }, + proxyTest{ + method: "GET", + localPath: "/proxy/alertmanager/dummy/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", + upstreamURI: "http://localhost:9093/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", + code: 404, + response: "404 page not found", + }, +} + +func TestProxy(t *testing.T) { + r := ginTestEngine() + setupRouterProxyHandlers(r, &alertmanager.Alertmanager{ + URI: "http://localhost:9093", + Timeout: time.Second * 5, + Name: "dummy", + ProxyRequests: true, + }) + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + for _, testCase := range proxyTests { + httpmock.Reset() + if testCase.upstreamURI != "" { + httpmock.RegisterResponder(testCase.method, testCase.upstreamURI, httpmock.NewStringResponder(testCase.code, testCase.response)) + } + req, _ := http.NewRequest(testCase.method, testCase.localPath, nil) + resp := newCloseNotifyingRecorder() + r.ServeHTTP(resp, req) + if resp.Code != testCase.code { + t.Errorf("%s %s proxied to %s returned status %d while %d was expected", + testCase.method, testCase.localPath, testCase.upstreamURI, resp.Code, testCase.code) + } + body := resp.Body.String() + if body != testCase.response { + t.Errorf("%s %s proxied to %s returned content '%s' while '%s' was expected", + testCase.method, testCase.localPath, testCase.upstreamURI, body, testCase.response) + } + } +} From 3eb28b3a6f030cad61b9ed115c322d5246fa4b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Sat, 6 Jan 2018 17:19:05 -0800 Subject: [PATCH 2/8] Use functional options pattern when creating Alertmanager instances --- internal/alertmanager/dedup_test.go | 2 +- internal/alertmanager/upstream.go | 15 +++++++++++++-- internal/filters/filter_test.go | 2 +- main.go | 7 ++++++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/internal/alertmanager/dedup_test.go b/internal/alertmanager/dedup_test.go index f102660c2..039f9ec9e 100644 --- a/internal/alertmanager/dedup_test.go +++ b/internal/alertmanager/dedup_test.go @@ -16,7 +16,7 @@ import ( func init() { log.SetLevel(log.ErrorLevel) for i, uri := range mock.ListAllMockURIs() { - alertmanager.NewAlertmanager(fmt.Sprintf("dedup-mock-%d", i), uri, time.Second, false) + alertmanager.NewAlertmanager(fmt.Sprintf("dedup-mock-%d", i), uri, time.Second) } } diff --git a/internal/alertmanager/upstream.go b/internal/alertmanager/upstream.go index e6b025772..6965e59ea 100644 --- a/internal/alertmanager/upstream.go +++ b/internal/alertmanager/upstream.go @@ -14,8 +14,7 @@ var ( upstreams = map[string]*Alertmanager{} ) -// NewAlertmanager creates a new Alertmanager instance -func NewAlertmanager(name, uri string, timeout time.Duration, proxyRequests bool) error { +func newAlertmanager(name, uri string, timeout time.Duration, proxyRequests bool) error { if _, found := upstreams[name]; found { return fmt.Errorf("Alertmanager upstream '%s' already exist", name) } @@ -49,6 +48,18 @@ func NewAlertmanager(name, uri string, timeout time.Duration, proxyRequests bool return nil } +// NewAlertmanager creates a new Alertmanager instance, unsee clients will talk +// to directly to it without unsee proxying any request +func NewAlertmanager(name, uri string, timeout time.Duration) error { + return newAlertmanager(name, uri, timeout, false) +} + +// NewProxiedAlertmanager creates a new proxied Alertmanager instance, unsee +// clients will talk to it via unsee +func NewProxiedAlertmanager(name, uri string, timeout time.Duration) error { + return newAlertmanager(name, uri, timeout, false) +} + // GetAlertmanagers returns a list of all defined Alertmanager instances func GetAlertmanagers() []*Alertmanager { ams := []*Alertmanager{} diff --git a/internal/filters/filter_test.go b/internal/filters/filter_test.go index 819608218..9f39d8225 100644 --- a/internal/filters/filter_test.go +++ b/internal/filters/filter_test.go @@ -485,7 +485,7 @@ var tests = []filterTest{ func TestFilters(t *testing.T) { log.SetLevel(log.ErrorLevel) - err := alertmanager.NewAlertmanager("test", "http://localhost", time.Second, false) + err := alertmanager.NewAlertmanager("test", "http://localhost", time.Second) am := alertmanager.GetAlertmanagerByName("test") if err != nil { t.Error(err) diff --git a/main.go b/main.go index 056e57ffe..45552beb5 100644 --- a/main.go +++ b/main.go @@ -60,7 +60,12 @@ func setupRouter(router *gin.Engine) { func setupUpstreams() { for _, s := range config.Config.Alertmanager.Servers { - err := alertmanager.NewAlertmanager(s.Name, s.URI, s.Timeout, s.Proxy) + var err error + if s.Proxy { + err = alertmanager.NewProxiedAlertmanager(s.Name, s.URI, s.Timeout) + } else { + err = alertmanager.NewAlertmanager(s.Name, s.URI, s.Timeout) + } if err != nil { log.Fatalf("Failed to configure Alertmanager '%s' with URI '%s': %s", s.Name, s.URI, err) } From db0d86f339584336bdc6bfa4339aa2c431680c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Sat, 6 Jan 2018 17:28:03 -0800 Subject: [PATCH 3/8] Don't modify path in the proxy, strip prefix in the handler --- proxy.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/proxy.go b/proxy.go index bc56adf80..e563e8cb4 100644 --- a/proxy.go +++ b/proxy.go @@ -5,7 +5,6 @@ import ( "net/http" "net/http/httputil" "net/url" - "strings" "github.com/cloudflare/unsee/internal/alertmanager" "github.com/cloudflare/unsee/internal/config" @@ -28,7 +27,6 @@ func NewAlertmanagerProxy(alertmanager *alertmanager.Alertmanager) (*httputil.Re Director: func(req *http.Request) { req.URL.Scheme = upstreamURL.Scheme req.URL.Host = upstreamURL.Host - req.URL.Path = strings.TrimPrefix(req.URL.Path, proxyPathPrefix(alertmanager.Name)) // drop Accept-Encoding header so we always get uncompressed reponses from // upstream, there's a gzip middleware that's global so we don't want it // to gzip twice @@ -50,7 +48,11 @@ func setupRouterProxyHandlers(router *gin.Engine, alertmanager *alertmanager.Ale if err != nil { return err } - router.POST(fmt.Sprintf("%s/api/v1/silences", proxyPathPrefix(alertmanager.Name)), gin.WrapH(proxy)) - router.DELETE(fmt.Sprintf("%s/api/v1/silence/*id", proxyPathPrefix(alertmanager.Name)), gin.WrapH(proxy)) + router.POST( + fmt.Sprintf("%s/api/v1/silences", proxyPathPrefix(alertmanager.Name)), + gin.WrapH(http.StripPrefix(proxyPathPrefix(alertmanager.Name), proxy))) + router.DELETE( + fmt.Sprintf("%s/api/v1/silence/*id", proxyPathPrefix(alertmanager.Name)), + gin.WrapH(http.StripPrefix(proxyPathPrefix(alertmanager.Name), proxy))) return nil } From 6aec8bf5d1551e8d12dad70b09ed2ee71e50b6fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Sat, 6 Jan 2018 17:35:02 -0800 Subject: [PATCH 4/8] Cleanup path setup a bit --- proxy.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/proxy.go b/proxy.go index e563e8cb4..e11f83d3d 100644 --- a/proxy.go +++ b/proxy.go @@ -17,6 +17,10 @@ func proxyPathPrefix(name string) string { return fmt.Sprintf("%sproxy/alertmanager/%s", config.Config.Listen.Prefix, name) } +func proxyPath(name, path string) string { + return fmt.Sprintf("%s%s", proxyPathPrefix(name), path) +} + // NewAlertmanagerProxy creates a proxy instance for given alertmanager instance func NewAlertmanagerProxy(alertmanager *alertmanager.Alertmanager) (*httputil.ReverseProxy, error) { upstreamURL, err := url.Parse(alertmanager.URI) @@ -49,10 +53,10 @@ func setupRouterProxyHandlers(router *gin.Engine, alertmanager *alertmanager.Ale return err } router.POST( - fmt.Sprintf("%s/api/v1/silences", proxyPathPrefix(alertmanager.Name)), + proxyPath(alertmanager.Name, "/api/v1/silences"), gin.WrapH(http.StripPrefix(proxyPathPrefix(alertmanager.Name), proxy))) router.DELETE( - fmt.Sprintf("%s/api/v1/silence/*id", proxyPathPrefix(alertmanager.Name)), + proxyPath(alertmanager.Name, "/api/v1/silence/*id"), gin.WrapH(http.StripPrefix(proxyPathPrefix(alertmanager.Name), proxy))) return nil } From 2176c44ba71998405ba8a119c1b75cd4b033610b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Sat, 6 Jan 2018 17:39:19 -0800 Subject: [PATCH 5/8] Drop names in slices --- proxy_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/proxy_test.go b/proxy_test.go index e40c97b0c..30ded1702 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -42,14 +42,14 @@ type proxyTest struct { var proxyTests = []proxyTest{ // valid alertmanager and methods - proxyTest{ + { method: "POST", localPath: "/proxy/alertmanager/dummy/api/v1/silences", upstreamURI: "http://localhost:9093/api/v1/silences", code: 200, response: "{\"status\":\"success\",\"data\":{\"silenceId\":\"d8a61ca8-ee2e-4076-999f-276f1e986bf3\"}}", }, - proxyTest{ + { method: "DELETE", localPath: "/proxy/alertmanager/dummy/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", upstreamURI: "http://localhost:9093/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", @@ -57,14 +57,14 @@ var proxyTests = []proxyTest{ response: "{\"status\":\"success\"}", }, // invalid alertmanager name - proxyTest{ + { method: "POST", localPath: "/proxy/alertmanager/INVALID/api/v1/silences", upstreamURI: "", code: 404, response: "404 page not found", }, - proxyTest{ + { method: "DELETE", localPath: "/proxy/alertmanager/INVALID/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", upstreamURI: "http://localhost:9093/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", @@ -72,14 +72,14 @@ var proxyTests = []proxyTest{ response: "404 page not found", }, // valid alertmanager name, but invalid method - proxyTest{ + { method: "GET", localPath: "/proxy/alertmanager/dummy/api/v1/silences", upstreamURI: "", code: 404, response: "404 page not found", }, - proxyTest{ + { method: "GET", localPath: "/proxy/alertmanager/dummy/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", upstreamURI: "http://localhost:9093/api/v1/silence/d8a61ca8-ee2e-4076-999f-276f1e986bf3", From c8b7b3bcd08889f70c2336168a7f9dc1cd903754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Mon, 8 Jan 2018 13:30:46 -0800 Subject: [PATCH 6/8] Implement functional options, this time for real --- internal/alertmanager/dedup_test.go | 3 +- internal/alertmanager/upstream.go | 59 ++++++++++++++++++----------- internal/filters/filter_test.go | 2 +- main.go | 6 +-- 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/internal/alertmanager/dedup_test.go b/internal/alertmanager/dedup_test.go index 039f9ec9e..6fb5f90b9 100644 --- a/internal/alertmanager/dedup_test.go +++ b/internal/alertmanager/dedup_test.go @@ -16,7 +16,8 @@ import ( func init() { log.SetLevel(log.ErrorLevel) for i, uri := range mock.ListAllMockURIs() { - alertmanager.NewAlertmanager(fmt.Sprintf("dedup-mock-%d", i), uri, time.Second) + name := fmt.Sprintf("dedup-mock-%d", i) + alertmanager.NewAlertmanager(name, uri, alertmanager.WithRequestTimeout(time.Second)) } } diff --git a/internal/alertmanager/upstream.go b/internal/alertmanager/upstream.go index 6965e59ea..7b832c0f1 100644 --- a/internal/alertmanager/upstream.go +++ b/internal/alertmanager/upstream.go @@ -10,11 +10,15 @@ import ( log "github.com/sirupsen/logrus" ) +// Option allows to pass functional options to NewAlertmanager() +type Option func(am *Alertmanager) + var ( upstreams = map[string]*Alertmanager{} ) -func newAlertmanager(name, uri string, timeout time.Duration, proxyRequests bool) error { +// NewAlertmanager creates a new Alertmanager instance +func NewAlertmanager(name, uri string, opts ...Option) error { if _, found := upstreams[name]; found { return fmt.Errorf("Alertmanager upstream '%s' already exist", name) } @@ -25,16 +29,15 @@ func newAlertmanager(name, uri string, timeout time.Duration, proxyRequests bool } } - upstreams[name] = &Alertmanager{ - URI: uri, - Timeout: timeout, - Name: name, - ProxyRequests: proxyRequests, - lock: sync.RWMutex{}, - alertGroups: []models.AlertGroup{}, - silences: map[string]models.Silence{}, - colors: models.LabelsColorMap{}, - autocomplete: []models.Autocomplete{}, + am := &Alertmanager{ + URI: uri, + Timeout: time.Second * 10, + Name: name, + lock: sync.RWMutex{}, + alertGroups: []models.AlertGroup{}, + silences: map[string]models.Silence{}, + colors: models.LabelsColorMap{}, + autocomplete: []models.Autocomplete{}, metrics: alertmanagerMetrics{ errors: map[string]float64{ labelValueErrorsAlerts: 0, @@ -43,23 +46,17 @@ func newAlertmanager(name, uri string, timeout time.Duration, proxyRequests bool }, } + for _, opt := range opts { + opt(am) + } + + upstreams[name] = am + log.Infof("[%s] Configured Alertmanager source at %s", name, uri) return nil } -// NewAlertmanager creates a new Alertmanager instance, unsee clients will talk -// to directly to it without unsee proxying any request -func NewAlertmanager(name, uri string, timeout time.Duration) error { - return newAlertmanager(name, uri, timeout, false) -} - -// NewProxiedAlertmanager creates a new proxied Alertmanager instance, unsee -// clients will talk to it via unsee -func NewProxiedAlertmanager(name, uri string, timeout time.Duration) error { - return newAlertmanager(name, uri, timeout, false) -} - // GetAlertmanagers returns a list of all defined Alertmanager instances func GetAlertmanagers() []*Alertmanager { ams := []*Alertmanager{} @@ -78,3 +75,19 @@ func GetAlertmanagerByName(name string) *Alertmanager { } return nil } + +// WithProxy option can be passed to NewAlertmanager in order to enable request +// proxying for unsee clients +func WithProxy(proxied bool) Option { + return func(am *Alertmanager) { + am.ProxyRequests = proxied + } +} + +// WithRequestTimeout option can be passed to NewAlertmanager in order to set +// a custom timeout for Alertmanager upstream requests +func WithRequestTimeout(timeout time.Duration) Option { + return func(am *Alertmanager) { + am.Timeout = timeout + } +} diff --git a/internal/filters/filter_test.go b/internal/filters/filter_test.go index 9f39d8225..6bbd50cf9 100644 --- a/internal/filters/filter_test.go +++ b/internal/filters/filter_test.go @@ -485,7 +485,7 @@ var tests = []filterTest{ func TestFilters(t *testing.T) { log.SetLevel(log.ErrorLevel) - err := alertmanager.NewAlertmanager("test", "http://localhost", time.Second) + err := alertmanager.NewAlertmanager("test", "http://localhost", alertmanager.WithRequestTimeout(time.Second)) am := alertmanager.GetAlertmanagerByName("test") if err != nil { t.Error(err) diff --git a/main.go b/main.go index 45552beb5..d1c86b5c3 100644 --- a/main.go +++ b/main.go @@ -61,11 +61,7 @@ func setupRouter(router *gin.Engine) { func setupUpstreams() { for _, s := range config.Config.Alertmanager.Servers { var err error - if s.Proxy { - err = alertmanager.NewProxiedAlertmanager(s.Name, s.URI, s.Timeout) - } else { - err = alertmanager.NewAlertmanager(s.Name, s.URI, s.Timeout) - } + err = alertmanager.NewAlertmanager(s.Name, s.URI, alertmanager.WithRequestTimeout(s.Timeout), alertmanager.WithProxy(s.Proxy)) if err != nil { log.Fatalf("Failed to configure Alertmanager '%s' with URI '%s': %s", s.Name, s.URI, err) } From efda91f542084cee5ad1392b3f99522a6c4dd766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Mon, 8 Jan 2018 13:51:38 -0800 Subject: [PATCH 7/8] Rename Alertmanager.Timeout -> Alertmanager.RequestTimeout --- internal/alertmanager/models.go | 12 ++++++------ internal/alertmanager/upstream.go | 18 +++++++++--------- proxy_test.go | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/alertmanager/models.go b/internal/alertmanager/models.go index 809ba0edc..ce5399119 100644 --- a/internal/alertmanager/models.go +++ b/internal/alertmanager/models.go @@ -29,9 +29,9 @@ type alertmanagerMetrics struct { // Alertmanager represents Alertmanager upstream instance type Alertmanager struct { - URI string `json:"uri"` - Timeout time.Duration `json:"timeout"` - Name string `json:"name"` + URI string `json:"uri"` + RequestTimeout time.Duration `json:"timeout"` + Name string `json:"name"` // whenever this instance should be proxied ProxyRequests bool // lock protects data access while updating @@ -56,7 +56,7 @@ func (am *Alertmanager) detectVersion() string { return defaultVersion } ver := alertmanagerVersion{} - err = transport.ReadJSON(url, am.Timeout, &ver) + err = transport.ReadJSON(url, am.RequestTimeout, &ver) if err != nil { log.Errorf("[%s] %s request failed: %s", am.Name, url, err.Error()) return defaultVersion @@ -92,7 +92,7 @@ func (am *Alertmanager) pullSilences(version string) error { } start := time.Now() - silences, err := mapper.GetSilences(am.URI, am.Timeout) + silences, err := mapper.GetSilences(am.URI, am.RequestTimeout) if err != nil { return err } @@ -135,7 +135,7 @@ func (am *Alertmanager) pullAlerts(version string) error { } start := time.Now() - groups, err := mapper.GetAlerts(am.URI, am.Timeout) + groups, err := mapper.GetAlerts(am.URI, am.RequestTimeout) if err != nil { return err } diff --git a/internal/alertmanager/upstream.go b/internal/alertmanager/upstream.go index 7b832c0f1..1c52a4e54 100644 --- a/internal/alertmanager/upstream.go +++ b/internal/alertmanager/upstream.go @@ -30,14 +30,14 @@ func NewAlertmanager(name, uri string, opts ...Option) error { } am := &Alertmanager{ - URI: uri, - Timeout: time.Second * 10, - Name: name, - lock: sync.RWMutex{}, - alertGroups: []models.AlertGroup{}, - silences: map[string]models.Silence{}, - colors: models.LabelsColorMap{}, - autocomplete: []models.Autocomplete{}, + URI: uri, + RequestTimeout: time.Second * 10, + Name: name, + lock: sync.RWMutex{}, + alertGroups: []models.AlertGroup{}, + silences: map[string]models.Silence{}, + colors: models.LabelsColorMap{}, + autocomplete: []models.Autocomplete{}, metrics: alertmanagerMetrics{ errors: map[string]float64{ labelValueErrorsAlerts: 0, @@ -88,6 +88,6 @@ func WithProxy(proxied bool) Option { // a custom timeout for Alertmanager upstream requests func WithRequestTimeout(timeout time.Duration) Option { return func(am *Alertmanager) { - am.Timeout = timeout + am.RequestTimeout = timeout } } diff --git a/proxy_test.go b/proxy_test.go index 30ded1702..47621d63d 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -91,10 +91,10 @@ var proxyTests = []proxyTest{ func TestProxy(t *testing.T) { r := ginTestEngine() setupRouterProxyHandlers(r, &alertmanager.Alertmanager{ - URI: "http://localhost:9093", - Timeout: time.Second * 5, - Name: "dummy", - ProxyRequests: true, + URI: "http://localhost:9093", + RequestTimeout: time.Second * 5, + Name: "dummy", + ProxyRequests: true, }) httpmock.Activate() From 983c7f50eea8c742c3f03a6e12f6daad642a311b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Mierzwa?= Date: Mon, 8 Jan 2018 14:28:28 -0800 Subject: [PATCH 8/8] Refactor NewAlertmanager() to not register new instance by default This makes it easier to test code --- internal/alertmanager/dedup_test.go | 3 ++- internal/alertmanager/upstream.go | 29 ++++++++++++++++------------- internal/filters/filter_test.go | 6 +----- main.go | 4 ++-- proxy_test.go | 13 +++++++------ 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/internal/alertmanager/dedup_test.go b/internal/alertmanager/dedup_test.go index 6fb5f90b9..1d7e64430 100644 --- a/internal/alertmanager/dedup_test.go +++ b/internal/alertmanager/dedup_test.go @@ -17,7 +17,8 @@ func init() { log.SetLevel(log.ErrorLevel) for i, uri := range mock.ListAllMockURIs() { name := fmt.Sprintf("dedup-mock-%d", i) - alertmanager.NewAlertmanager(name, uri, alertmanager.WithRequestTimeout(time.Second)) + am := alertmanager.NewAlertmanager(name, uri, alertmanager.WithRequestTimeout(time.Second)) + alertmanager.RegisterAlertmanager(am) } } diff --git a/internal/alertmanager/upstream.go b/internal/alertmanager/upstream.go index 1c52a4e54..285b973c5 100644 --- a/internal/alertmanager/upstream.go +++ b/internal/alertmanager/upstream.go @@ -18,17 +18,7 @@ var ( ) // NewAlertmanager creates a new Alertmanager instance -func NewAlertmanager(name, uri string, opts ...Option) error { - if _, found := upstreams[name]; found { - return fmt.Errorf("Alertmanager upstream '%s' already exist", name) - } - - for _, am := range upstreams { - if am.URI == uri { - return fmt.Errorf("Alertmanager upstream '%s' already collects from '%s'", am.Name, am.URI) - } - } - +func NewAlertmanager(name, uri string, opts ...Option) *Alertmanager { am := &Alertmanager{ URI: uri, RequestTimeout: time.Second * 10, @@ -50,10 +40,23 @@ func NewAlertmanager(name, uri string, opts ...Option) error { opt(am) } - upstreams[name] = am + return am +} - log.Infof("[%s] Configured Alertmanager source at %s", name, uri) +// RegisterAlertmanager will add an Alertmanager instance to the list of +// instances used when pulling alerts from upstreams +func RegisterAlertmanager(am *Alertmanager) error { + if _, found := upstreams[am.Name]; found { + return fmt.Errorf("Alertmanager upstream '%s' already exist", am.Name) + } + for _, existingAM := range upstreams { + if existingAM.URI == am.URI { + return fmt.Errorf("Alertmanager upstream '%s' already collects from '%s'", existingAM.Name, existingAM.URI) + } + } + upstreams[am.Name] = am + log.Infof("[%s] Configured Alertmanager source at %s (proxied: %v)", am.Name, am.URI, am.ProxyRequests) return nil } diff --git a/internal/filters/filter_test.go b/internal/filters/filter_test.go index 6bbd50cf9..c329afb2d 100644 --- a/internal/filters/filter_test.go +++ b/internal/filters/filter_test.go @@ -485,11 +485,7 @@ var tests = []filterTest{ func TestFilters(t *testing.T) { log.SetLevel(log.ErrorLevel) - err := alertmanager.NewAlertmanager("test", "http://localhost", alertmanager.WithRequestTimeout(time.Second)) - am := alertmanager.GetAlertmanagerByName("test") - if err != nil { - t.Error(err) - } + am := alertmanager.NewAlertmanager("test", "http://localhost", alertmanager.WithRequestTimeout(time.Second)) for _, ft := range tests { alert := models.Alert(ft.Alert) if &ft.Silence != nil { diff --git a/main.go b/main.go index d1c86b5c3..b517c54dd 100644 --- a/main.go +++ b/main.go @@ -60,8 +60,8 @@ func setupRouter(router *gin.Engine) { func setupUpstreams() { for _, s := range config.Config.Alertmanager.Servers { - var err error - err = alertmanager.NewAlertmanager(s.Name, s.URI, alertmanager.WithRequestTimeout(s.Timeout), alertmanager.WithProxy(s.Proxy)) + am := alertmanager.NewAlertmanager(s.Name, s.URI, alertmanager.WithRequestTimeout(s.Timeout), alertmanager.WithProxy(s.Proxy)) + err := alertmanager.RegisterAlertmanager(am) if err != nil { log.Fatalf("Failed to configure Alertmanager '%s' with URI '%s': %s", s.Name, s.URI, err) } diff --git a/proxy_test.go b/proxy_test.go index 47621d63d..18ed8c9f1 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -90,12 +90,13 @@ var proxyTests = []proxyTest{ func TestProxy(t *testing.T) { r := ginTestEngine() - setupRouterProxyHandlers(r, &alertmanager.Alertmanager{ - URI: "http://localhost:9093", - RequestTimeout: time.Second * 5, - Name: "dummy", - ProxyRequests: true, - }) + am := alertmanager.NewAlertmanager( + "dummy", + "http://localhost:9093", + alertmanager.WithRequestTimeout(time.Second*5), + alertmanager.WithProxy(true), + ) + setupRouterProxyHandlers(r, am) httpmock.Activate() defer httpmock.DeactivateAndReset()