fix(tests): improve test coverage

cmd/karma/tests/testscript/035_log_requests.txt

@@ -1,6 +1,6 @@
 # Works in simple mode when single --alertmanager.uri flag is passed
 
-exec sh -ex ./test.sh &
+exec sh -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8035 --log.requests
 ! stdout .
 stderr 'msg="Request completed" address=127.0.0.1:[0-9]+ bytes=[0-9]+ code=200 duration=".+" method=GET path=/alerts.json'

cmd/karma/tests/testscript/068_sentry.txt

@@ -1,6 +1,6 @@
 # Configures sentry when enabled
 
-exec sh -ex ./test.sh &
+exec sh -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --log.config=true --config.file=karma.yaml --listen.address=127.0.0.1 --listen.port=8068
 ! stdout .
 cmp stderr stderr.txt

cmd/karma/tests/testscript/069_simple_config.txt

@@ -1,6 +1,6 @@
 # Works in simple mode when single --alertmanager.uri flag is passed
 
-exec sh -ex ./test.sh &
+exec sh -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8069
 ! stdout .
 cmp stderr stderr.txt

cmd/karma/tests/testscript/073_pid_file_remove_error.txt

@@ -1,6 +1,6 @@
 # Fails is we cannot remove a PID file
 
-exec sh -ex ./test.sh &
+exec sh -x ./test.sh &
 karma.bin-should-fail --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8073
 ! stdout .
 cmp stderr stderr.txt

cmd/karma/tests/testscript/076_basicAuth_uri.txt

@@ -1,6 +1,6 @@
 # Basic Auth headers are passed to the UI in the API response
 
-exec sh -ex ./test.sh &
+exec sh -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://foo:bar@127.0.0.1 --listen.address=127.0.0.1 --listen.port=8076
 wait
 stdout '"headers":{"Authorization":"Basic Zm9vOmJhcg=="}'

cmd/karma/tests/testscript/083_metrics.txt

@@ -0,0 +1,140 @@
+# GET /metrics returns same list of metrics
+
+exec sh -x ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8083
+! stdout .
+cmp stderr stderr.txt
+cmp curl.txt metrics.txt
+
+-- metrics.txt --
+# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"}
+go_gc_duration_seconds{quantile="0.25"}
+go_gc_duration_seconds{quantile="0.5"}
+go_gc_duration_seconds{quantile="0.75"}
+go_gc_duration_seconds{quantile="1"}
+go_gc_duration_seconds_sum
+go_gc_duration_seconds_count
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+go_info{version="go1.15.3"}
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes
+# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
+# TYPE go_memstats_alloc_bytes_total counter
+go_memstats_alloc_bytes_total
+# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
+# TYPE go_memstats_buck_hash_sys_bytes gauge
+go_memstats_buck_hash_sys_bytes
+# HELP go_memstats_frees_total Total number of frees.
+# TYPE go_memstats_frees_total counter
+go_memstats_frees_total
+# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started.
+# TYPE go_memstats_gc_cpu_fraction gauge
+go_memstats_gc_cpu_fraction
+# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
+# TYPE go_memstats_gc_sys_bytes gauge
+go_memstats_gc_sys_bytes
+# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
+# TYPE go_memstats_heap_alloc_bytes gauge
+go_memstats_heap_alloc_bytes
+# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
+# TYPE go_memstats_heap_idle_bytes gauge
+go_memstats_heap_idle_bytes
+# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
+# TYPE go_memstats_heap_inuse_bytes gauge
+go_memstats_heap_inuse_bytes
+# HELP go_memstats_heap_objects Number of allocated objects.
+# TYPE go_memstats_heap_objects gauge
+go_memstats_heap_objects
+# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
+# TYPE go_memstats_heap_released_bytes gauge
+go_memstats_heap_released_bytes
+# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
+# TYPE go_memstats_heap_sys_bytes gauge
+go_memstats_heap_sys_bytes
+# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
+# TYPE go_memstats_last_gc_time_seconds gauge
+go_memstats_last_gc_time_seconds
+# HELP go_memstats_lookups_total Total number of pointer lookups.
+# TYPE go_memstats_lookups_total counter
+go_memstats_lookups_total
+# HELP go_memstats_mallocs_total Total number of mallocs.
+# TYPE go_memstats_mallocs_total counter
+go_memstats_mallocs_total
+# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
+# TYPE go_memstats_mcache_inuse_bytes gauge
+go_memstats_mcache_inuse_bytes
+# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
+# TYPE go_memstats_mcache_sys_bytes gauge
+go_memstats_mcache_sys_bytes
+# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
+# TYPE go_memstats_mspan_inuse_bytes gauge
+go_memstats_mspan_inuse_bytes
+# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
+# TYPE go_memstats_mspan_sys_bytes gauge
+go_memstats_mspan_sys_bytes
+# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
+# TYPE go_memstats_next_gc_bytes gauge
+go_memstats_next_gc_bytes
+# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
+# TYPE go_memstats_other_sys_bytes gauge
+go_memstats_other_sys_bytes
+# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
+# TYPE go_memstats_stack_inuse_bytes gauge
+go_memstats_stack_inuse_bytes
+# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
+# TYPE go_memstats_stack_sys_bytes gauge
+go_memstats_stack_sys_bytes
+# HELP go_memstats_sys_bytes Number of bytes obtained from system.
+# TYPE go_memstats_sys_bytes gauge
+go_memstats_sys_bytes
+# HELP go_threads Number of OS threads created.
+# TYPE go_threads gauge
+go_threads
+# HELP karma_alertmanager_errors_total Total number of errors encounter when requesting data from Alertmanager API
+# TYPE karma_alertmanager_errors_total counter
+karma_alertmanager_errors_total{alertmanager="default",endpoint="alerts"}
+karma_alertmanager_errors_total{alertmanager="default",endpoint="silences"}
+# HELP karma_alertmanager_up 1 if last call to Alertmanager API succeeded
+# TYPE karma_alertmanager_up gauge
+karma_alertmanager_up{alertmanager="default"}
+# HELP karma_collect_cycles_total Total number of alert collection cycles run
+# TYPE karma_collect_cycles_total counter
+karma_collect_cycles_total{alertmanager="default"}
+# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
+# TYPE promhttp_metric_handler_requests_in_flight gauge
+promhttp_metric_handler_requests_in_flight
+# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
+# TYPE promhttp_metric_handler_requests_total counter
+promhttp_metric_handler_requests_total{code="200"}
+promhttp_metric_handler_requests_total{code="500"}
+promhttp_metric_handler_requests_total{code="503"}
+-- stderr.txt --
+level=info msg="Version: dev"
+level=info msg="Configured Alertmanager source" name=default proxy=false readonly=false uri=http://127.0.0.1
+level=info msg="Writing PID file" path=karma.pid
+level=info msg="Initial Alertmanager collection"
+level=info msg="Pulling latest alerts and silences from Alertmanager"
+level=info msg="Collecting alerts and silences" alertmanager=default
+level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
+level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
+level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default
+level=info msg="Collection completed"
+level=info msg="Done, starting HTTP server"
+level=info msg="Starting HTTP server" address=127.0.0.1:8083
+level=info msg="Shutting down HTTP server"
+level=info msg="HTTP server shut down"
+level=info msg="Removing PID file" path=karma.pid
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 5
+curl -s http://localhost:8083/metrics | grep -v 'process_' | sed -E s_' ([0-9]+|[0-9]+\.[0-9]+|[0-9]+\.[0-9]+e(\+|\-)[0-9]+)$'_''_g > curl.txt
+cat karma.pid | xargs kill

cmd/karma/tests/testscript/084_tls_client.txt

@@ -0,0 +1,187 @@
+# Correctly passes TLS client config to the upstream
+
+exec sh -x ./tls.sh
+exec sh -x ./test.sh &
+exec sh -c 'I=0 ; while [ ! -f alertmanager.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done'
+karma.bin-should-work --pid-file=karma.pid
+! stdout .
+stderr 'level=info msg="Upstream version" alertmanager=client-auth version=0.21.0'
+stderr 'level=info msg="Got silences" alertmanager=client-auth duration=.+ silences=0'
+stderr 'level=info msg="Detecting ticket links in silences" alertmanager=client-auth silences=0'
+stderr 'level=info msg="Collected alert groups" alertmanager=client-auth duration=.+ groups=0'
+stderr 'level=info msg="Deduplicating alert groups" alertmanager=client-auth groups=0'
+stderr 'level=info msg="Processing deduplicated alert groups" alertmanager=client-auth groups=0'
+stderr 'level=info msg="Merging autocomplete hints" alertmanager=client-auth hints=0'
+stderr 'level=info msg="Collection completed"'
+
+-- alertmanager.conf --
+[req]
+distinguished_name = DN
+x509_extensions = SAN
+[DN]
+CN = 127.0.0.1
+[SAN]
+basicConstraints     = CA:FALSE
+subjectKeyIdentifier = hash
+keyUsage             = digitalSignature, keyEncipherment
+extendedKeyUsage     = clientAuth, serverAuth
+subjectAltName       = @alt_names
+[alt_names]
+DNS.1 = localhost
+IP.1  = 127.0.0.1
+
+-- karma.conf --
+[req]
+distinguished_name = DN
+req_extensions = SAN
+[DN]
+CN = karma
+[SAN]
+basicConstraints     = CA:FALSE
+subjectKeyIdentifier = hash
+keyUsage             = digitalSignature, keyEncipherment
+extendedKeyUsage     = clientAuth, serverAuth
+subjectAltName       = @alt_names
+[alt_names]
+DNS.1 = karma
+
+-- tls.sh --
+openssl ecparam -genkey -name secp256r1 | openssl ec -out ca.key
+openssl req -new -x509 -days 7 -key ca.key -out ca.pem -subj "/C=CI/ST=CI/L=CI/O=CI/CN=FakeCA"
+
+openssl ecparam -genkey -name secp256r1 | openssl ec -out alertmanager.key
+openssl req -new -key alertmanager.key -out alertmanager.csr -subj "/C=CI/ST=CI/L=CI/O=CI/CN=127.0.0.1" -config alertmanager.conf -extensions SAN
+openssl x509 -req -days 7 -extfile alertmanager.conf -extensions SAN -in alertmanager.csr -CA ca.pem -CAkey ca.key -set_serial 01 -out alertmanager.pem
+openssl x509 -in alertmanager.pem -text
+
+openssl ecparam -genkey -name secp256r1 | openssl ec -out karma.key
+openssl req -new -key karma.key -out karma.csr -subj "/C=CI/ST=CI/L=CI/O=CI/CN=karma" -config karma.conf -extensions SAN
+openssl x509 -req -days 7 -extfile karma.conf -extensions SAN -in karma.csr -CA ca.pem -CAkey ca.key -set_serial 02 -out karma.pem
+openssl x509 -in karma.pem -text
+
+-- test.sh --
+#!/bin/sh
+
+env GOCACHE=$TMPDIR go run alertmanager.go &
+
+I=0
+while [ ! -f karma.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done
+
+sleep 5
+cat karma.pid | xargs kill
+cat alertmanager.pid | xargs kill
+
+-- karma.yaml --
+alertmanager:
+  interval: 1h
+  servers:
+    - name: client-auth
+      uri: https://127.0.0.1:9084
+      timeout: 10s
+      tls:
+        ca: ca.pem
+        cert: karma.pem
+        key: karma.key
+listen:
+  address: 127.0.0.1
+  port: 8084
+
+-- alertmanager.go --
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"syscall"
+	"time"
+)
+
+func metrics(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	io.WriteString(w, `alertmanager_build_info{version="0.21.0"} 1
+`)
+}
+
+func status(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	io.WriteString(w, `
+{
+    "cluster": {
+        "name": "01EB67XCFES27NAFAGSW48NAHC",
+        "peers": [
+            {
+                "address": "172.17.0.2:9094",
+                "name": "01EB67XCFES27NAFAGSW48NAHC"
+            }
+        ],
+        "status": "ready"
+    },
+    "versionInfo": {
+        "version": "0.21.0"
+    }
+}`)
+}
+
+func empty(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	io.WriteString(w, "[]")
+}
+
+func main() {
+	pid := os.Getpid()
+	err := ioutil.WriteFile("alertmanager.pid", []byte(strconv.Itoa(pid)), 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	http.HandleFunc("/metrics", metrics)
+	http.HandleFunc("/api/v2/status", status)
+	http.HandleFunc("/api/v2/silences", empty)
+	http.HandleFunc("/api/v2/alerts/groups", empty)
+
+	caCert, err := ioutil.ReadFile("ca.pem")
+	if err != nil {
+		log.Fatal(err)
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+
+	tlsConfig := &tls.Config{
+		ClientCAs:          caCertPool,
+		ClientAuth:         tls.RequireAndVerifyClientCert,
+		InsecureSkipVerify: true,
+	}
+
+	listener, err := net.Listen("tcp", "127.0.0.1:9084")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	server := &http.Server{
+		Addr:      "127.0.0.1:9084",
+		TLSConfig: tlsConfig,
+	}
+
+	go func() {
+		err := server.ServeTLS(listener, "alertmanager.pem", "alertmanager.key")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	stop := make(chan os.Signal, 1)
+	signal.Notify(stop, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	<-stop
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	server.Shutdown(ctx)
+}

internal/mock/mock.go

@@ -1,7 +1,6 @@
 package mock
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
@@ -21,13 +20,7 @@ func GetAbsoluteMockPath(filename string, version string) string {
 // GetMockResponder returns a httpmock.Responder for given file/version
 func GetMockResponder(url string, version string, filename string) httpmock.Responder {
 	fullPath := GetAbsoluteMockPath(filename, version)
-	mockJSON, err := ioutil.ReadFile(fullPath)
-	if err != nil {
-		panic(fmt.Errorf("failed to read %s: %s", fullPath, err))
-	}
-	if len(mockJSON) == 0 {
-		panic(fmt.Errorf("empty mock file '%s'", fullPath))
-	}
+	mockJSON, _ := ioutil.ReadFile(fullPath)
 	return httpmock.NewBytesResponder(200, mockJSON)
 }
 
@@ -41,11 +34,7 @@ func ListAllMocks() []string {
 	_, f, _, _ := runtime.Caller(0)
 	cwd := filepath.Dir(f)
 
-	dirents, err := ioutil.ReadDir(cwd)
-	if err != nil {
-		panic(err)
-	}
-
+	dirents, _ := ioutil.ReadDir(cwd)
 	dirs := []string{}
 	for _, dirent := range dirents {
 		if dirent.IsDir() {