feat(backend): add tls options for history requests

Fixes #3707

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## [next]
+
+### Added
+
+- `tls` options to `history:rewrite` rules, allowing customising TLS options
+  for requests made by karma to Prometheus servers when querying alert
+  history, #3707.
+
 ## v0.93
 
 ### Fixed

cmd/karma/alert_history.go

@@ -16,9 +16,12 @@ import (
 	"github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
+	"github.com/rs/zerolog/log"
+
+	"github.com/prymitive/karma/internal/alertmanager"
 	"github.com/prymitive/karma/internal/config"
 	"github.com/prymitive/karma/internal/slices"
-	"github.com/rs/zerolog/log"
+	uriUtil "github.com/prymitive/karma/internal/uri"
 )
 
 type AlertHistoryPayload struct {
@@ -138,7 +141,7 @@ func newHistoryPoller(queueSize int, queryTimeout time.Duration) *historyPoller
 
 func (hp *historyPoller) run(workers int) {
 	wg := sync.WaitGroup{}
-	for w := 1; w < workers; w++ {
+	for w := 1; w <= workers; w++ {
 		w := w
 		wg.Add(1)
 		go func() {
@@ -206,7 +209,15 @@ func (hp *historyPoller) startWorker(wid int) {
 			j.result <- historyQueryResult{values: v.values, err: nil}
 			continue
 		}
-		values, err := countAlerts(sourceURI, hp.queryTimeout, j.labels)
+		transport, err := rewriteTransport(config.Config.History.Rewrite, j.uri)
+		if err != nil {
+			log.Warn().
+				Int("worker", wid).
+				Str("uri", sourceURI).
+				Err(err).
+				Msg("Error while configuring HTTP transport for history request")
+		}
+		values, err := countAlerts(sourceURI, hp.queryTimeout, transport, j.labels)
 		if err != nil {
 			log.Error().
 				Err(err).
@@ -251,14 +262,33 @@ func rewriteSource(rules []config.HistoryRewrite, uri string) string {
 	return uri
 }
 
-func countAlerts(uri string, timeout time.Duration, labels map[string]string) (ret []OffsetSample, err error) {
+func rewriteTransport(rules []config.HistoryRewrite, uri string) (http.RoundTripper, error) {
+	// trim trailing / to ensure all URIs are without a /
+	uri = strings.TrimSuffix(uri, "/")
+	for _, rule := range rules {
+		if !rule.SourceRegex.MatchString(uri) {
+			continue
+		}
+		if rule.TLS.CA != "" || rule.TLS.Cert != "" || rule.TLS.InsecureSkipVerify {
+			transport, err := alertmanager.NewHTTPTransport(rule.TLS.CA, rule.TLS.Cert, rule.TLS.Key, rule.TLS.InsecureSkipVerify)
+			if err != nil {
+				return http.DefaultTransport, fmt.Errorf("failed to create HTTP transport for '%s': %w", uriUtil.SanitizeURI(uri), err)
+			}
+			return transport, nil
+		}
+	}
+
+	return http.DefaultTransport, nil
+}
+
+func countAlerts(uri string, timeout time.Duration, transport http.RoundTripper, labels map[string]string) (ret []OffsetSample, err error) {
 	if uri == "" {
 		return
 	}
 
 	client, err := api.NewClient(api.Config{
 		Address:      uri,
-		RoundTripper: http.DefaultTransport,
+		RoundTripper: transport,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to create Prometheus API client: %w", err)

cmd/karma/alert_history_test.go

@@ -464,6 +464,72 @@ func TestAlertHistory(t *testing.T) {
 				},
 			},
 		},
+		{
+			mocks: []mock{
+				{
+					method: "GET",
+					uri:    regexp.MustCompile("^http://localhost:9100/api/v1/labels"),
+					responder: httpmock.NewJsonResponderOrPanic(200, prometheusAPIV1Labels{
+						Status: "success",
+						Data:   []string{"alertname", "instance", "job"},
+					}),
+				},
+				{
+					method: "POST",
+					uri:    regexp.MustCompile("^http://localhost:9100/api/v1/query_range"),
+					responder: httpmock.NewJsonResponderOrPanic(200, prometheusAPIV1QueryRange{
+						Status: "success",
+						Data: generateV1Matrix(
+							[]seriesValues{
+								{
+									metric: model.Metric{
+										"alertname": "Fake Alert",
+									},
+									values: generateIntSlice(0, 1, 24),
+								},
+							}, time.Hour),
+					}),
+				},
+			},
+			config: cfg{
+				enabled: true,
+				timeout: time.Second * 5,
+				workers: 5,
+				rewrite: []config.HistoryRewrite{
+					{
+						SourceRegex: regex.MustCompileAnchored("http://(.+):1111"),
+						URI:         "http://$1:9100",
+						TLS: config.AlertmanagerTLS{
+							InsecureSkipVerify: true,
+						},
+					},
+					{
+						SourceRegex: regex.MustCompileAnchored("foo"),
+						URI:         "",
+					},
+					{
+						SourceRegex: regex.MustCompileAnchored("http://(.+):909[0-9]"),
+						URI:         "http://$1:9100",
+						TLS: config.AlertmanagerTLS{
+							CA:                 "/xxx/yyy/bbb/foo.crt",
+							InsecureSkipVerify: true,
+						},
+					},
+				},
+			},
+			queries: []historyQuery{
+				{
+					payload: generateHistoryPayload(AlertHistoryPayload{
+						Sources: []string{"http://localhost:9090/", "http://localhost:9091/", "http://localhost:1111/"},
+						Labels:  map[string]string{"alertname": "Fake Alert", "cluster": "prod"},
+					}),
+					code: 200,
+					response: AlertHistoryResponse{
+						Samples: generateHistorySamples(generateIntSlice(0, 3, 24), time.Hour),
+					},
+				},
+			},
+		},
 	}
 
 	defer func() {

cmd/karma/tests/testscript/060_log_full_config_file.txt

@@ -178,6 +178,18 @@ level=info msg="  timeout: 1h0m0s"
 level=info msg="  rewrite:"
 level=info msg="    - source: http://(.+).example.com"
 level=info msg="      uri: https://prod-$1.example.com"
+level=info msg="      tls:"
+level=info msg="        ca: \"\""
+level=info msg="        cert: \"\""
+level=info msg="        key: \"\""
+level=info msg="        insecureSkipVerify: false"
+level=info msg="    - source: (.+)"
+level=info msg="      uri: $1"
+level=info msg="      tls:"
+level=info msg="        ca: /etc/ca.pem"
+level=info msg="        cert: /etc/server.pem"
+level=info msg="        key: /etc/server.key"
+level=info msg="        insecureSkipVerify: true"
 level=info msg="karma:"
 level=info msg="  name: karma-demo"
 level=info msg="labels:"
@@ -397,6 +409,13 @@ history:
   rewrite:
     - source: "http://(.+).example.com"
       uri: "https://prod-$1.example.com"
+    - source: "(.+)"
+      uri: "$1"
+      tls:
+        ca: /etc/ca.pem
+        cert: /etc/server.pem
+        key: /etc/server.key
+        insecureSkipVerify: true
 karma:
   name: karma-demo
 labels:

cmd/karma/tests/testscript/103_alert_history_tls.txt

@@ -0,0 +1,166 @@
+# GET /history.json
+
+exec bash -x ./tls.sh
+exec bash -x ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --config.file=karma.yaml
+! stdout .
+cmp stderr stderr.txt
+
+-- stderr.txt --
+level=info msg="Reading configuration file" path=karma.yaml
+level=info msg="Version: dev"
+level=info msg="Configured Alertmanager source" name=default proxy=false readonly=false uri=http://127.0.0.1
+level=info msg="Writing PID file" path=karma.pid
+level=info msg="Initial Alertmanager collection"
+level=info msg="Pulling latest alerts and silences from Alertmanager"
+level=info msg="Collecting alerts and silences" alertmanager=default
+level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
+level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
+level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
+level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
+level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
+level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
+level=info msg="Collection completed"
+level=info msg="Done, starting HTTP server"
+level=info msg="Starting HTTP server" address=127.0.0.1:8103
+level=info msg="Shutting down HTTP server"
+level=info msg="HTTP server shut down"
+level=info msg="Removing PID file" path=karma.pid
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: http://127.0.0.1
+listen:
+  address: 127.0.0.1
+  port: 8103
+history:
+  enabled: true
+  timeout: 10s
+  rewrite:
+    - source: '(.*)'
+      uri: '$1'
+      tls:
+        ca: ./ca.pem
+        insecureSkipVerify: false
+-- query.json --
+{
+    "sources": [
+        "https://127.0.0.1:9103",
+        "https://127.0.0.1:9103"
+    ],
+    "labels": {
+        "alertname": "Fake Alert"
+    }
+}
+-- prometheus.conf --
+[req]
+distinguished_name = DN
+x509_extensions = SAN
+[DN]
+CN = 127.0.0.1
+[SAN]
+basicConstraints     = CA:FALSE
+subjectKeyIdentifier = hash
+keyUsage             = digitalSignature, keyEncipherment
+extendedKeyUsage     = clientAuth, serverAuth
+subjectAltName       = @alt_names
+[alt_names]
+DNS.1 = localhost
+IP.1  = 127.0.0.1
+-- test.sh --
+env GOCACHE=$TMPDIR go run prometheus.go &
+
+I=0
+while [ ! -f prometheus.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done
+
+I=0
+while [ ! -f karma.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done
+
+sleep 5
+curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8103/history.json
+cat karma.pid | xargs kill
+cat prometheus.pid | xargs kill
+
+-- tls.sh --
+openssl ecparam -genkey -name secp256r1 | openssl ec -out ca.key
+openssl req -new -x509 -days 7 -key ca.key -out ca.pem -subj "/C=CI/ST=CI/L=CI/O=CI/CN=FakeCA"
+
+openssl ecparam -genkey -name secp256r1 | openssl ec -out prometheus.key
+openssl req -new -key prometheus.key -out prometheus.csr -subj "/C=CI/ST=CI/L=CI/O=CI/CN=127.0.0.1" -config prometheus.conf -extensions SAN
+openssl x509 -req -days 7 -extfile prometheus.conf -extensions SAN -in prometheus.csr -CA ca.pem -CAkey ca.key -set_serial 01 -out prometheus.pem
+openssl x509 -in prometheus.pem -text
+
+-- prometheus.go --
+package main
+
+import (
+	"context"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+)
+
+func labelNames(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	io.WriteString(w, `{
+    "status": "success",
+    "data": ["alertname"]
+}`)
+}
+
+func query(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	io.WriteString(w, `{
+	"status": "success",
+	"data": {
+		"resultType": "matrix",
+		"result": [
+			{
+				"metric": {},
+				"values": []
+			}
+		]
+	}
+}`)
+}
+
+func main() {
+	pid := os.Getpid()
+	err := os.WriteFile("prometheus.pid", []byte(strconv.Itoa(pid)), 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	http.HandleFunc("/api/v1/labels", labelNames)
+	http.HandleFunc("/api/v1/query_range", query)
+
+	listener, err := net.Listen("tcp", "127.0.0.1:9103")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	server := &http.Server{
+		Addr: "127.0.0.1:9103",
+	}
+
+	go func() {
+		err := server.ServeTLS(listener, "prometheus.pem", "prometheus.key")
+		if err != nil {
+			log.Printf("Serve returned error: %v", err)
+		}
+	}()
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	<-stop
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	server.Shutdown(ctx)
+}

cmd/karma/tests/testscript/104_alert_history_tls_bad.txt

@@ -0,0 +1,143 @@
+# GET /history.json
+
+exec bash -x ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --config.file=karma.yaml
+! stdout .
+cmp stderr stderr.txt
+
+-- stderr.txt --
+level=info msg="Reading configuration file" path=karma.yaml
+level=info msg="Version: dev"
+level=info msg="Configured Alertmanager source" name=default proxy=false readonly=false uri=http://127.0.0.1
+level=info msg="Writing PID file" path=karma.pid
+level=info msg="Initial Alertmanager collection"
+level=info msg="Pulling latest alerts and silences from Alertmanager"
+level=info msg="Collecting alerts and silences" alertmanager=default
+level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
+level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
+level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
+level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
+level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
+level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
+level=info msg="Collection completed"
+level=info msg="Done, starting HTTP server"
+level=info msg="Starting HTTP server" address=127.0.0.1:8104
+level=warn msg="Error while configuring HTTP transport for history request" error="failed to create HTTP transport for 'http://127.0.0.1:9104': open /xxx/yyy/ca.pem: no such file or directory" uri=http://127.0.0.1:9104 worker=1
+level=info msg="Shutting down HTTP server"
+level=info msg="HTTP server shut down"
+level=info msg="Removing PID file" path=karma.pid
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: http://127.0.0.1
+listen:
+  address: 127.0.0.1
+  port: 8104
+history:
+  enabled: true
+  workers: 1
+  timeout: 10s
+  rewrite:
+    - source: '(.*)'
+      uri: '$1'
+      tls:
+        ca: /xxx/yyy/ca.pem
+        insecureSkipVerify: true
+-- query.json --
+{
+    "sources": [
+        "http://127.0.0.1:9104",
+        "http://127.0.0.1:9104"
+    ],
+    "labels": {
+        "alertname": "Fake Alert"
+    }
+}
+-- test.sh --
+env GOCACHE=$TMPDIR go run prometheus.go &
+
+I=0
+while [ ! -f prometheus.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done
+
+I=0
+while [ ! -f karma.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done
+
+sleep 5
+curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8104/history.json
+cat karma.pid | xargs kill
+cat prometheus.pid | xargs kill
+
+-- prometheus.go --
+package main
+
+import (
+	"context"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+)
+
+func labelNames(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	io.WriteString(w, `{
+    "status": "success",
+    "data": ["alertname"]
+}`)
+}
+
+func query(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	io.WriteString(w, `{
+	"status": "success",
+	"data": {
+		"resultType": "matrix",
+		"result": [
+			{
+				"metric": {},
+				"values": []
+			}
+		]
+	}
+}`)
+}
+
+func main() {
+	pid := os.Getpid()
+	err := os.WriteFile("prometheus.pid", []byte(strconv.Itoa(pid)), 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	http.HandleFunc("/api/v1/labels", labelNames)
+	http.HandleFunc("/api/v1/query_range", query)
+
+	listener, err := net.Listen("tcp", "127.0.0.1:9104")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	server := &http.Server{
+		Addr: "127.0.0.1:9104",
+	}
+
+	go func() {
+		err := server.Serve(listener)
+		if err != nil {
+			log.Printf("Serve returned error: %v", err)
+		}
+	}()
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	<-stop
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	server.Shutdown(ctx)
+}

cmd/karma/tests/testscript/105_alert_history_invalid_workers.txt

@@ -0,0 +1,13 @@
+karma.bin-should-fail --check-config
+! stdout .
+cmp stderr stderr.txt
+
+-- stderr.txt --
+level=error msg="Execution failed" error="history.workers must be >= 1"
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: https://127.0.0.1:9093
+history:
+  workers: 0

docs/CONFIGURATION.md

@@ -758,6 +758,11 @@ history:
   rewrite:
     - source: regex
       uri: string
+      tls:
+        ca: string
+        cert: string
+        key: string
+        insecureSkipVerify: bool
 ```
 
 - `enabled` - enable alert history UI and backend query support
@@ -766,9 +771,9 @@ history:
   one outgoing HTTP request, more workers allows to handle more concurrent
   queries if you have a large number of Prometheus servers sending alerts
 - `rewrite` - list of source rewrite rules applied before any request is send
-  to remote Prometheus. Rewrite rules can be used to modify URI used by karma
-  when connecting to Prometheus API if `source` field in alert uses addresses
-  not reachable from karma.
+  to remote Prometheus. Rewrite rules can be used to modify URI or TLS settings
+  used by karma when connecting to Prometheus API if `source` field in alert
+  uses addresses not reachable from karma.
   All regexes are anchored, `${N}` syntax can be used for capture groups.
   You can rewrite uri to an empty string to disable connecting to that
   specific Prometheus instance.
@@ -814,6 +819,17 @@ history:
       uri: ''
 ```
 
+Example with rewrite rule that configures TLS settings without modifying URI:
+
+```YAML
+history:
+  rewrite:
+    - source: '(.*)'
+      uri: '$1'
+      tls:
+        insecureSkipVerify: true
+```
+
 ### Karma
 
 `karma` section allows configuring miscellaneous internal options.

internal/config/config.go

@@ -439,6 +439,9 @@ func (config *configSchema) Read(flags *pflag.FlagSet) (string, error) {
 		return "", fmt.Errorf("listen.tls.cert must be set when listen.tls.key is set")
 	}
 
+	if config.History.Workers < 1 {
+		return "", fmt.Errorf("history.workers must be >= 1")
+	}
 	for i := 0; i < len(config.History.Rewrite); i++ {
 		config.History.Rewrite[i].SourceRegex, err = regex.CompileAnchored(config.History.Rewrite[i].Source)
 		if err != nil {

internal/config/models.go

@@ -61,9 +61,10 @@ type AuthorizationGroup struct {
 }
 
 type HistoryRewrite struct {
-	Source      string         `yaml:"source"`
-	SourceRegex *regexp.Regexp `yaml:"-"`
-	URI         string         `yaml:"uri"`
+	Source      string          `yaml:"source"`
+	SourceRegex *regexp.Regexp  `yaml:"-"`
+	URI         string          `yaml:"uri"`
+	TLS         AlertmanagerTLS `yaml:"tls" koanf:"tls"`
 }
 
 type configSchema struct {