diff --git a/CHANGELOG.md b/CHANGELOG.md index 3565ea548..007e34482 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [next] + +### Added + +- `tls` options to `history:rewrite` rules, allowing customising TLS options + for requests made by karma to Prometheus servers when querying alert + history, #3707. + ## v0.93 ### Fixed diff --git a/cmd/karma/alert_history.go b/cmd/karma/alert_history.go index 9878d842b..1f4b8fd9e 100644 --- a/cmd/karma/alert_history.go +++ b/cmd/karma/alert_history.go @@ -16,9 +16,12 @@ import ( "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" + "github.com/rs/zerolog/log" + + "github.com/prymitive/karma/internal/alertmanager" "github.com/prymitive/karma/internal/config" "github.com/prymitive/karma/internal/slices" - "github.com/rs/zerolog/log" + uriUtil "github.com/prymitive/karma/internal/uri" ) type AlertHistoryPayload struct { @@ -138,7 +141,7 @@ func newHistoryPoller(queueSize int, queryTimeout time.Duration) *historyPoller func (hp *historyPoller) run(workers int) { wg := sync.WaitGroup{} - for w := 1; w < workers; w++ { + for w := 1; w <= workers; w++ { w := w wg.Add(1) go func() { @@ -206,7 +209,15 @@ func (hp *historyPoller) startWorker(wid int) { j.result <- historyQueryResult{values: v.values, err: nil} continue } - values, err := countAlerts(sourceURI, hp.queryTimeout, j.labels) + transport, err := rewriteTransport(config.Config.History.Rewrite, j.uri) + if err != nil { + log.Warn(). + Int("worker", wid). + Str("uri", sourceURI). + Err(err). + Msg("Error while configuring HTTP transport for history request") + } + values, err := countAlerts(sourceURI, hp.queryTimeout, transport, j.labels) if err != nil { log.Error(). Err(err). @@ -251,14 +262,33 @@ func rewriteSource(rules []config.HistoryRewrite, uri string) string { return uri } -func countAlerts(uri string, timeout time.Duration, labels map[string]string) (ret []OffsetSample, err error) { +func rewriteTransport(rules []config.HistoryRewrite, uri string) (http.RoundTripper, error) { + // trim trailing / to ensure all URIs are without a / + uri = strings.TrimSuffix(uri, "/") + for _, rule := range rules { + if !rule.SourceRegex.MatchString(uri) { + continue + } + if rule.TLS.CA != "" || rule.TLS.Cert != "" || rule.TLS.InsecureSkipVerify { + transport, err := alertmanager.NewHTTPTransport(rule.TLS.CA, rule.TLS.Cert, rule.TLS.Key, rule.TLS.InsecureSkipVerify) + if err != nil { + return http.DefaultTransport, fmt.Errorf("failed to create HTTP transport for '%s': %w", uriUtil.SanitizeURI(uri), err) + } + return transport, nil + } + } + + return http.DefaultTransport, nil +} + +func countAlerts(uri string, timeout time.Duration, transport http.RoundTripper, labels map[string]string) (ret []OffsetSample, err error) { if uri == "" { return } client, err := api.NewClient(api.Config{ Address: uri, - RoundTripper: http.DefaultTransport, + RoundTripper: transport, }) if err != nil { return nil, fmt.Errorf("failed to create Prometheus API client: %w", err) diff --git a/cmd/karma/alert_history_test.go b/cmd/karma/alert_history_test.go index d33589b14..6c3cbfe4f 100644 --- a/cmd/karma/alert_history_test.go +++ b/cmd/karma/alert_history_test.go @@ -464,6 +464,72 @@ func TestAlertHistory(t *testing.T) { }, }, }, + { + mocks: []mock{ + { + method: "GET", + uri: regexp.MustCompile("^http://localhost:9100/api/v1/labels"), + responder: httpmock.NewJsonResponderOrPanic(200, prometheusAPIV1Labels{ + Status: "success", + Data: []string{"alertname", "instance", "job"}, + }), + }, + { + method: "POST", + uri: regexp.MustCompile("^http://localhost:9100/api/v1/query_range"), + responder: httpmock.NewJsonResponderOrPanic(200, prometheusAPIV1QueryRange{ + Status: "success", + Data: generateV1Matrix( + []seriesValues{ + { + metric: model.Metric{ + "alertname": "Fake Alert", + }, + values: generateIntSlice(0, 1, 24), + }, + }, time.Hour), + }), + }, + }, + config: cfg{ + enabled: true, + timeout: time.Second * 5, + workers: 5, + rewrite: []config.HistoryRewrite{ + { + SourceRegex: regex.MustCompileAnchored("http://(.+):1111"), + URI: "http://$1:9100", + TLS: config.AlertmanagerTLS{ + InsecureSkipVerify: true, + }, + }, + { + SourceRegex: regex.MustCompileAnchored("foo"), + URI: "", + }, + { + SourceRegex: regex.MustCompileAnchored("http://(.+):909[0-9]"), + URI: "http://$1:9100", + TLS: config.AlertmanagerTLS{ + CA: "/xxx/yyy/bbb/foo.crt", + InsecureSkipVerify: true, + }, + }, + }, + }, + queries: []historyQuery{ + { + payload: generateHistoryPayload(AlertHistoryPayload{ + Sources: []string{"http://localhost:9090/", "http://localhost:9091/", "http://localhost:1111/"}, + Labels: map[string]string{"alertname": "Fake Alert", "cluster": "prod"}, + }), + code: 200, + response: AlertHistoryResponse{ + Samples: generateHistorySamples(generateIntSlice(0, 3, 24), time.Hour), + }, + }, + }, + }, } defer func() { diff --git a/cmd/karma/tests/testscript/060_log_full_config_file.txt b/cmd/karma/tests/testscript/060_log_full_config_file.txt index c93e40acf..f42b880d8 100644 --- a/cmd/karma/tests/testscript/060_log_full_config_file.txt +++ b/cmd/karma/tests/testscript/060_log_full_config_file.txt @@ -178,6 +178,18 @@ level=info msg=" timeout: 1h0m0s" level=info msg=" rewrite:" level=info msg=" - source: http://(.+).example.com" level=info msg=" uri: https://prod-$1.example.com" +level=info msg=" tls:" +level=info msg=" ca: \"\"" +level=info msg=" cert: \"\"" +level=info msg=" key: \"\"" +level=info msg=" insecureSkipVerify: false" +level=info msg=" - source: (.+)" +level=info msg=" uri: $1" +level=info msg=" tls:" +level=info msg=" ca: /etc/ca.pem" +level=info msg=" cert: /etc/server.pem" +level=info msg=" key: /etc/server.key" +level=info msg=" insecureSkipVerify: true" level=info msg="karma:" level=info msg=" name: karma-demo" level=info msg="labels:" @@ -397,6 +409,13 @@ history: rewrite: - source: "http://(.+).example.com" uri: "https://prod-$1.example.com" + - source: "(.+)" + uri: "$1" + tls: + ca: /etc/ca.pem + cert: /etc/server.pem + key: /etc/server.key + insecureSkipVerify: true karma: name: karma-demo labels: diff --git a/cmd/karma/tests/testscript/103_alert_history_tls.txt b/cmd/karma/tests/testscript/103_alert_history_tls.txt new file mode 100644 index 000000000..390b786fa --- /dev/null +++ b/cmd/karma/tests/testscript/103_alert_history_tls.txt @@ -0,0 +1,166 @@ +# GET /history.json + +exec bash -x ./tls.sh +exec bash -x ./test.sh & +karma.bin-should-work --pid-file=karma.pid --config.file=karma.yaml +! stdout . +cmp stderr stderr.txt + +-- stderr.txt -- +level=info msg="Reading configuration file" path=karma.yaml +level=info msg="Version: dev" +level=info msg="Configured Alertmanager source" name=default proxy=false readonly=false uri=http://127.0.0.1 +level=info msg="Writing PID file" path=karma.pid +level=info msg="Initial Alertmanager collection" +level=info msg="Pulling latest alerts and silences from Alertmanager" +level=info msg="Collecting alerts and silences" alertmanager=default +level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics +level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1 +level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2 +level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics +level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1 +level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2 +level=info msg="Collection completed" +level=info msg="Done, starting HTTP server" +level=info msg="Starting HTTP server" address=127.0.0.1:8103 +level=info msg="Shutting down HTTP server" +level=info msg="HTTP server shut down" +level=info msg="Removing PID file" path=karma.pid +-- karma.yaml -- +alertmanager: + servers: + - name: default + uri: http://127.0.0.1 +listen: + address: 127.0.0.1 + port: 8103 +history: + enabled: true + timeout: 10s + rewrite: + - source: '(.*)' + uri: '$1' + tls: + ca: ./ca.pem + insecureSkipVerify: false +-- query.json -- +{ + "sources": [ + "https://127.0.0.1:9103", + "https://127.0.0.1:9103" + ], + "labels": { + "alertname": "Fake Alert" + } +} +-- prometheus.conf -- +[req] +distinguished_name = DN +x509_extensions = SAN +[DN] +CN = 127.0.0.1 +[SAN] +basicConstraints = CA:FALSE +subjectKeyIdentifier = hash +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = localhost +IP.1 = 127.0.0.1 +-- test.sh -- +env GOCACHE=$TMPDIR go run prometheus.go & + +I=0 +while [ ! -f prometheus.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done + +I=0 +while [ ! -f karma.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done + +sleep 5 +curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8103/history.json +cat karma.pid | xargs kill +cat prometheus.pid | xargs kill + +-- tls.sh -- +openssl ecparam -genkey -name secp256r1 | openssl ec -out ca.key +openssl req -new -x509 -days 7 -key ca.key -out ca.pem -subj "/C=CI/ST=CI/L=CI/O=CI/CN=FakeCA" + +openssl ecparam -genkey -name secp256r1 | openssl ec -out prometheus.key +openssl req -new -key prometheus.key -out prometheus.csr -subj "/C=CI/ST=CI/L=CI/O=CI/CN=127.0.0.1" -config prometheus.conf -extensions SAN +openssl x509 -req -days 7 -extfile prometheus.conf -extensions SAN -in prometheus.csr -CA ca.pem -CAkey ca.key -set_serial 01 -out prometheus.pem +openssl x509 -in prometheus.pem -text + +-- prometheus.go -- +package main + +import ( + "context" + "io" + "log" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" +) + +func labelNames(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{ + "status": "success", + "data": ["alertname"] +}`) +} + +func query(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [] + } + ] + } +}`) +} + +func main() { + pid := os.Getpid() + err := os.WriteFile("prometheus.pid", []byte(strconv.Itoa(pid)), 0644) + if err != nil { + log.Fatal(err) + } + + http.HandleFunc("/api/v1/labels", labelNames) + http.HandleFunc("/api/v1/query_range", query) + + listener, err := net.Listen("tcp", "127.0.0.1:9103") + if err != nil { + log.Fatal(err) + } + + server := &http.Server{ + Addr: "127.0.0.1:9103", + } + + go func() { + err := server.ServeTLS(listener, "prometheus.pem", "prometheus.key") + if err != nil { + log.Printf("Serve returned error: %v", err) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + <-stop + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + server.Shutdown(ctx) +} diff --git a/cmd/karma/tests/testscript/104_alert_history_tls_bad.txt b/cmd/karma/tests/testscript/104_alert_history_tls_bad.txt new file mode 100644 index 000000000..95f7ccc40 --- /dev/null +++ b/cmd/karma/tests/testscript/104_alert_history_tls_bad.txt @@ -0,0 +1,143 @@ +# GET /history.json + +exec bash -x ./test.sh & +karma.bin-should-work --pid-file=karma.pid --config.file=karma.yaml +! stdout . +cmp stderr stderr.txt + +-- stderr.txt -- +level=info msg="Reading configuration file" path=karma.yaml +level=info msg="Version: dev" +level=info msg="Configured Alertmanager source" name=default proxy=false readonly=false uri=http://127.0.0.1 +level=info msg="Writing PID file" path=karma.pid +level=info msg="Initial Alertmanager collection" +level=info msg="Pulling latest alerts and silences from Alertmanager" +level=info msg="Collecting alerts and silences" alertmanager=default +level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics +level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1 +level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2 +level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics +level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1 +level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2 +level=info msg="Collection completed" +level=info msg="Done, starting HTTP server" +level=info msg="Starting HTTP server" address=127.0.0.1:8104 +level=warn msg="Error while configuring HTTP transport for history request" error="failed to create HTTP transport for 'http://127.0.0.1:9104': open /xxx/yyy/ca.pem: no such file or directory" uri=http://127.0.0.1:9104 worker=1 +level=info msg="Shutting down HTTP server" +level=info msg="HTTP server shut down" +level=info msg="Removing PID file" path=karma.pid +-- karma.yaml -- +alertmanager: + servers: + - name: default + uri: http://127.0.0.1 +listen: + address: 127.0.0.1 + port: 8104 +history: + enabled: true + workers: 1 + timeout: 10s + rewrite: + - source: '(.*)' + uri: '$1' + tls: + ca: /xxx/yyy/ca.pem + insecureSkipVerify: true +-- query.json -- +{ + "sources": [ + "http://127.0.0.1:9104", + "http://127.0.0.1:9104" + ], + "labels": { + "alertname": "Fake Alert" + } +} +-- test.sh -- +env GOCACHE=$TMPDIR go run prometheus.go & + +I=0 +while [ ! -f prometheus.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done + +I=0 +while [ ! -f karma.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done + +sleep 5 +curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8104/history.json +cat karma.pid | xargs kill +cat prometheus.pid | xargs kill + +-- prometheus.go -- +package main + +import ( + "context" + "io" + "log" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" +) + +func labelNames(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{ + "status": "success", + "data": ["alertname"] +}`) +} + +func query(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{ + "status": "success", + "data": { + "resultType": "matrix", + "result": [ + { + "metric": {}, + "values": [] + } + ] + } +}`) +} + +func main() { + pid := os.Getpid() + err := os.WriteFile("prometheus.pid", []byte(strconv.Itoa(pid)), 0644) + if err != nil { + log.Fatal(err) + } + + http.HandleFunc("/api/v1/labels", labelNames) + http.HandleFunc("/api/v1/query_range", query) + + listener, err := net.Listen("tcp", "127.0.0.1:9104") + if err != nil { + log.Fatal(err) + } + + server := &http.Server{ + Addr: "127.0.0.1:9104", + } + + go func() { + err := server.Serve(listener) + if err != nil { + log.Printf("Serve returned error: %v", err) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + <-stop + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + server.Shutdown(ctx) +} diff --git a/cmd/karma/tests/testscript/105_alert_history_invalid_workers.txt b/cmd/karma/tests/testscript/105_alert_history_invalid_workers.txt new file mode 100644 index 000000000..2f25c07f1 --- /dev/null +++ b/cmd/karma/tests/testscript/105_alert_history_invalid_workers.txt @@ -0,0 +1,13 @@ +karma.bin-should-fail --check-config +! stdout . +cmp stderr stderr.txt + +-- stderr.txt -- +level=error msg="Execution failed" error="history.workers must be >= 1" +-- karma.yaml -- +alertmanager: + servers: + - name: default + uri: https://127.0.0.1:9093 +history: + workers: 0 diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 1cddd006a..94c40112c 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -758,6 +758,11 @@ history: rewrite: - source: regex uri: string + tls: + ca: string + cert: string + key: string + insecureSkipVerify: bool ``` - `enabled` - enable alert history UI and backend query support @@ -766,9 +771,9 @@ history: one outgoing HTTP request, more workers allows to handle more concurrent queries if you have a large number of Prometheus servers sending alerts - `rewrite` - list of source rewrite rules applied before any request is send - to remote Prometheus. Rewrite rules can be used to modify URI used by karma - when connecting to Prometheus API if `source` field in alert uses addresses - not reachable from karma. + to remote Prometheus. Rewrite rules can be used to modify URI or TLS settings + used by karma when connecting to Prometheus API if `source` field in alert + uses addresses not reachable from karma. All regexes are anchored, `${N}` syntax can be used for capture groups. You can rewrite uri to an empty string to disable connecting to that specific Prometheus instance. @@ -814,6 +819,17 @@ history: uri: '' ``` +Example with rewrite rule that configures TLS settings without modifying URI: + +```YAML +history: + rewrite: + - source: '(.*)' + uri: '$1' + tls: + insecureSkipVerify: true +``` + ### Karma `karma` section allows configuring miscellaneous internal options. diff --git a/internal/config/config.go b/internal/config/config.go index 611229dd6..a337af58e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -439,6 +439,9 @@ func (config *configSchema) Read(flags *pflag.FlagSet) (string, error) { return "", fmt.Errorf("listen.tls.cert must be set when listen.tls.key is set") } + if config.History.Workers < 1 { + return "", fmt.Errorf("history.workers must be >= 1") + } for i := 0; i < len(config.History.Rewrite); i++ { config.History.Rewrite[i].SourceRegex, err = regex.CompileAnchored(config.History.Rewrite[i].Source) if err != nil { diff --git a/internal/config/models.go b/internal/config/models.go index 16fc41fcd..9b1c69393 100644 --- a/internal/config/models.go +++ b/internal/config/models.go @@ -61,9 +61,10 @@ type AuthorizationGroup struct { } type HistoryRewrite struct { - Source string `yaml:"source"` - SourceRegex *regexp.Regexp `yaml:"-"` - URI string `yaml:"uri"` + Source string `yaml:"source"` + SourceRegex *regexp.Regexp `yaml:"-"` + URI string `yaml:"uri"` + TLS AlertmanagerTLS `yaml:"tls" koanf:"tls"` } type configSchema struct {