fix(tests): refactor testscript tests

cmd/karma/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"html/template"
+	"io/ioutil"
 	"mime"
 	"net"
 	"net/http"
@@ -11,6 +12,7 @@ import (
 	"os/signal"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -54,6 +56,8 @@ var (
 	protectedEndpoints *gin.RouterGroup
 
 	silenceACLs = []*silenceACL{}
+
+	pidFile string
 )
 
 func getViewURL(sub string) string {
@@ -252,6 +256,7 @@ func mainSetup(errorHandling pflag.ErrorHandling) (*gin.Engine, error) {
 	f := pflag.NewFlagSet("karma", errorHandling)
 	printVersion := f.Bool("version", false, "Print version and exit")
 	validateConfig := f.Bool("check-config", false, "Validate configuration and exit")
+	f.StringVar(&pidFile, "pid-file", "", "If set PID of karma process will be written to this file")
 	config.SetupFlags(f)
 
 	err := f.Parse(os.Args[1:])
@@ -264,7 +269,11 @@ func mainSetup(errorHandling pflag.ErrorHandling) (*gin.Engine, error) {
 		return nil, nil
 	}
 
-	configFile := config.Config.Read(f)
+	configFile, err := config.Config.Read(f)
+	if err != nil {
+		return nil, err
+	}
+
 	err = setupLogger()
 	if err != nil {
 		return nil, err
@@ -368,13 +377,41 @@ func mainSetup(errorHandling pflag.ErrorHandling) (*gin.Engine, error) {
 	return router, nil
 }
 
-func main() {
-	router, err := mainSetup(pflag.ExitOnError)
+func writePidFile() error {
+	if pidFile != "" {
+		log.Infof("Writing PID file to %q", pidFile)
+		pid := os.Getpid()
+		err := ioutil.WriteFile(pidFile, []byte(strconv.Itoa(pid)), 0644)
+		if err != nil {
+			return fmt.Errorf("Failed to write a PID file: %s", err)
+		}
+	}
+	return nil
+}
+
+func removePidFile() error {
+	if pidFile != "" {
+		log.Infof("Removing PID file %q", pidFile)
+		err := os.Remove(pidFile)
+		if err != nil {
+			return fmt.Errorf("Failed to remove PID file: %s", err)
+		}
+	}
+	return nil
+}
+
+func serve(errorHandling pflag.ErrorHandling) error {
+	router, err := mainSetup(errorHandling)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	if router == nil {
-		return
+		return nil
+	}
+
+	err = writePidFile()
+	if err != nil {
+		return err
 	}
 
 	// before we start try to fetch data from Alertmanager
@@ -389,7 +426,7 @@ func main() {
 	listen := fmt.Sprintf("%s:%d", config.Config.Listen.Address, config.Config.Listen.Port)
 	listener, err := net.Listen("tcp", listen)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 	log.Infof("Listening on %s", listener.Addr())
 
@@ -409,7 +446,16 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	if err := httpServer.Shutdown(ctx); err != nil {
-		log.Fatalf("Shutdown failed: %s", err)
+		return fmt.Errorf("Shutdown failed: %s", err)
 	}
 	log.Info("HTTP server shut down")
+
+	return removePidFile()
+}
+
+func main() {
+	err := serve(pflag.ExitOnError)
+	if err != nil {
+		log.Fatal(err)
+	}
 }

cmd/karma/script_test.go

@@ -11,25 +11,13 @@ import (
 )
 
 func mainShoulFail() int {
-	var wasFatal bool
-	defer func() {
-		if r := recover(); r != nil {
-			wasFatal = true
-		}
-	}()
-	defer func() { log.StandardLogger().ExitFunc = nil }()
-	log.StandardLogger().ExitFunc = func(int) { wasFatal = true }
-
-	_, err := mainSetup(pflag.ContinueOnError)
+	err := serve(pflag.ContinueOnError)
 	if err != nil {
 		log.Error(err)
-	} else if wasFatal {
 		return 0
-	} else {
-		log.Error("No error logged")
-		return 100
 	}
-	return 0
+	log.Error("No error logged")
+	return 100
 }
 
 func mainShoulFailNoTimestamp() int {
@@ -40,7 +28,7 @@ func mainShoulFailNoTimestamp() int {
 }
 
 func mainShouldWork() int {
-	_, err := mainSetup(pflag.ContinueOnError)
+	err := serve(pflag.ContinueOnError)
 	if err != nil {
 		log.Error(err)
 		return 100

cmd/karma/tests/testscript/035_debug.txt

@@ -0,0 +1,12 @@
+# Passing --debug enables Gin debug mode
+
+exec sh -ex ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --log.format=text --log.config=false --debug --alertmanager.uri=http://localhost --listen.address=127.0.0.1 --listen.port=8035
+stdout '\[GIN-debug\] \[WARNING\] Running in "debug" mode. Switch to "release" mode in production.'
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+cat karma.pid | xargs kill

cmd/karma/tests/testscript/042_invalid_config_alertmanager_timeout.txt

@@ -1,7 +1,7 @@
 # Raises an error if we pass alertmanager.interval value that doesn't parse
 karma.bin-should-fail-no-timestamp --log.format=text --log.config=false --log.level=error --config.file karma.yaml
 ! stdout .
-stderr 'level=fatal msg=".* invalid duration \\"abc123\\""'
+stderr 'msg=".* invalid duration \\"abc123\\""'
 
 -- karma.yaml --
 alertmanager:

cmd/karma/tests/testscript/061_log_full_config_file_invalid_values.txt

@@ -0,0 +1,59 @@
+# Print out and compare logged config set via config file that includes invalid values
+karma.bin-should-fail-no-timestamp --config.file=karma.yaml --check-config
+! stdout .
+stderr 'Failed to unmarshal configuration: 12 error\(s\) decoding:'
+
+-- karma.yaml --
+alertmanager:
+  interval: jjs88
+  servers:
+    - name: ha1
+      uri: "http://localhost:9093"
+      timeout: bbb
+      proxy: YEs
+      cors:
+        credentials: foo
+    - name: ha2
+      uri: "http://localhost:9094"
+      timeout: 11
+      readonly: 1
+    - name: local
+      uri: http://localhost:9095
+      timeout: z
+      proxy: true
+      readonly: 0
+      headers:
+        - X-Auth-Test=some-token-or-other-string
+    - name: client-auth
+      uri: https://localhost:9096
+      timeout: 10s
+      tls:
+        ca: ca.pem
+        cert: cert.pem
+        key: key.pem
+alertAcknowledgement:
+  enabled: zzz
+  duration: 7m0s
+  author: karma
+  commentPrefix: ACK!
+annotations:
+  default:
+    hidden: z
+  hidden: {}
+  visible:
+    - visible
+filters:
+  default: []
+karma:
+  name: karma-demo
+log:
+  level: 123
+  format: foo
+ui:
+  refresh: 10sm
+  hideFiltersWhenIdle: z
+  colorTitlebar: yum
+  theme: x
+  minimalGroupWidth: abc4
+  alertsPerGroup: 5a
+  collapseGroups: collapsedOanMobile

cmd/karma/tests/testscript/063_missing_config_file.txt

@@ -0,0 +1,4 @@
+# Errors when config.file points to missing file
+karma.bin-should-fail --config.file=404.yaml
+! stdout .
+stderr 'msg="Failed to load configuration file \\"404.yaml\\": open 404.yaml: no such file or directory'

cmd/karma/tests/testscript/068_sentry.txt

@@ -0,0 +1,23 @@
+# Configures sentry when enabled
+
+exec sh -ex ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --log.format=text --log.config=true --config.file=karma.yaml --listen.address=127.0.0.1 --listen.port=8068
+! stdout .
+stderr 'msg="  private: secret"'
+stderr 'msg="  public: \\"123456789\\""'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: http://localhost:9093
+sentry:
+  private: secret
+  public: 123456789
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+cat karma.pid | xargs kill

cmd/karma/tests/testscript/069_simple_config.txt

@@ -0,0 +1,13 @@
+# Works in simple mode when single --alertmanager.uri flag is passed
+
+exec sh -ex ./test.sh &
+karma.bin-should-work --pid-file=karma.pid --log.format=text --log.config=false --alertmanager.uri=http://localhost --listen.address=127.0.0.1 --listen.port=8069
+! stdout .
+stderr 'msg="\[default\] Configured Alertmanager source at http://localhost \(proxied: false\, readonly: false\)"'
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+cat karma.pid | xargs kill

cmd/karma/tests/testscript/072_pid_file_write_error.txt

@@ -0,0 +1,5 @@
+# Fails is we cannot write a PID file
+
+karma.bin-should-fail --pid-file=/foo/bar/karma.pid --log.format=text --log.config=false --alertmanager.uri=http://localhost
+! stdout .
+stderr 'msg="Failed to write a PID file:'

cmd/karma/tests/testscript/073_pid_file_remove_error.txt

@@ -0,0 +1,15 @@
+# Fails is we cannot remove a PID file
+
+exec sh -ex ./test.sh &
+karma.bin-should-fail --pid-file=karma.pid --log.format=text --log.config=false --alertmanager.uri=http://localhost --listen.address=127.0.0.1 --listen.port=8073
+! stdout .
+stderr 'msg="Failed to remove PID file:'
+
+-- test.sh --
+#!/bin/sh
+
+while [ ! -f karma.pid ]; do sleep 1 ; done
+sleep 1
+PID=$(cat karma.pid)
+rm karma.pid
+kill $PID

cmd/karma/tests/testscript/debug.txt

@@ -1,3 +0,0 @@
-# Passing --debug enables Gin debug mode
-karma.bin-should-work --log.format=text --log.config=false --debug --alertmanager.uri=http://localhost
-stdout '\[GIN-debug\] \[WARNING\] Running in "debug" mode. Switch to "release" mode in production.'

cmd/karma/tests/testscript/log_full_config_file_invalid_values.txt

@@ -1,67 +0,0 @@
-# Print out and compare logged config set via config file that includes invalid values
-karma.bin-should-fail-no-timestamp --config.file=karma.yaml --check-config
-! stdout .
-cmp stderr expected.stderr
-
--- karma.yaml --
-alertmanager:
-  interval: jjs88
-  servers:
-    - name: ha1
-      uri: "http://localhost:9093"
-      timeout: bbb
-      proxy: YEs
-      cors:
-        credentials: foo
-    - name: ha2
-      uri: "http://localhost:9094"
-      timeout: 11
-      readonly: 1
-    - name: local
-      uri: http://localhost:9095
-      timeout: z
-      proxy: true
-      readonly: 0
-      headers:
-        - X-Auth-Test=some-token-or-other-string
-    - name: client-auth
-      uri: https://localhost:9096
-      timeout: 10s
-      tls:
-        ca: ca.pem
-        cert: cert.pem
-        key: key.pem
-alertAcknowledgement:
-  enabled: zzz
-  duration: 7m0s
-  author: karma
-  commentPrefix: ACK!
-annotations:
-  default:
-    hidden: z
-  hidden: {}
-  visible:
-    - visible
-filters:
-  default: []
-karma:
-  name: karma-demo
-log:
-  level: 123
-  format: foo
-ui:
-  refresh: 10sm
-  hideFiltersWhenIdle: z
-  colorTitlebar: yum
-  theme: x
-  minimalGroupWidth: abc4
-  alertsPerGroup: 5a
-  collapseGroups: collapsedOanMobile
-
--- expected.stderr --
-level=fatal msg="Failed to unmarshal configuration: 12 error(s) decoding:\n\n* 'Alertmanager.Servers[2].Headers[0]' expected a map, got 'string'\n* cannot parse 'Alertmanager.Servers[0].Proxy' as bool: strconv.ParseBool: parsing \"YEs\": invalid syntax\n* cannot parse 'Annotations.Default.Hidden' as bool: strconv.ParseBool: parsing \"z\": invalid syntax\n* cannot parse 'UI.alertsPerGroup' as int: strconv.ParseInt: parsing \"5a\": invalid syntax\n* cannot parse 'UI.colorTitlebar' as bool: strconv.ParseBool: parsing \"yum\": invalid syntax\n* cannot parse 'UI.hideFiltersWhenIdle' as bool: strconv.ParseBool: parsing \"z\": invalid syntax\n* cannot parse 'UI.minimalGroupWidth' as int: strconv.ParseInt: parsing \"abc4\": invalid syntax\n* cannot parse 'alertAcknowledgement.Enabled' as bool: strconv.ParseBool: parsing \"zzz\": invalid syntax\n* error decoding 'Alertmanager.Interval': time: invalid duration \"jjs88\"\n* error decoding 'Alertmanager.Servers[0].Timeout': time: invalid duration \"bbb\"\n* error decoding 'Alertmanager.Servers[2].Timeout': time: invalid duration \"z\"\n* error decoding 'UI.Refresh': time: unknown unit \"sm\" in duration \"10sm\""
-level=fatal msg="Invalid alertmanager.cors.credentials value '', allowed options: omit, inclue, same-origin"
-level=fatal msg="Invalid grid.sorting.order value '', allowed options: disabled, startsAt, label"
-level=fatal msg="Invalid ui.collapseGroups value '', allowed options: expanded, collapsed, collapsedOnMobile"
-level=fatal msg="Invalid ui.theme value '', allowed options: light, dark, auto"
-level=error msg="Unknown log level ''"

cmd/karma/tests/testscript/missing_config_file.txt

@@ -1,4 +0,0 @@
-# Errors when config.file points to missing file
-karma.bin-should-fail --config.file=404.yaml
-! stdout .
-stderr 'level=fatal msg="Failed to load configuration file \\"404.yaml\\": open 404.yaml: no such file or directory'

cmd/karma/tests/testscript/sentry.txt

@@ -1,14 +0,0 @@
-# Configures sentry when enabled
-karma.bin-should-work --log.format=text --log.config=true --config.file=karma.yaml
-! stdout .
-stderr 'msg="  private: secret"'
-stderr 'msg="  public: \\"123456789\\""'
-
--- karma.yaml --
-alertmanager:
-  servers:
-    - name: default
-      uri: http://localhost:9093
-sentry:
-  private: secret
-  public: 123456789

cmd/karma/tests/testscript/simple_config.txt

@@ -1,4 +0,0 @@
-# Works in simple mode when single --alertmanager.uri flag is passed
-karma.bin-should-work --log.format=text --log.config=false --alertmanager.uri=http://localhost
-! stdout .
-stderr 'msg="\[default\] Configured Alertmanager source at http://localhost \(proxied: false\, readonly: false\)"'

cmd/karma/views_test.go

@@ -37,7 +37,10 @@ func mockConfig() {
 
 	f := pflag.NewFlagSet(".", pflag.ExitOnError)
 	config.SetupFlags(f)
-	config.Config.Read(f)
+	_, err := config.Config.Read(f)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	if !upstreamSetup {
 		upstreamSetup = true

internal/alertmanager/benchmark_test.go

@@ -39,7 +39,7 @@ func BenchmarkDedupColors(b *testing.B) {
 
 	f := pflag.NewFlagSet(".", pflag.ExitOnError)
 	config.SetupFlags(f)
-	config.Config.Read(f)
+	_, _ = config.Config.Read(f)
 
 	if err := pullAlerts(); err != nil {
 		b.Error(err)

internal/alertmanager/dedup_test.go

@@ -52,7 +52,7 @@ func pullAlerts() error {
 func mockConfigRead() {
 	f := pflag.NewFlagSet(".", pflag.ExitOnError)
 	config.SetupFlags(f)
-	config.Config.Read(f)
+	_, _ = config.Config.Read(f)
 }
 
 func TestDedupAlerts(t *testing.T) {

internal/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"os"
 	"strings"
 	"time"
@@ -122,7 +123,7 @@ func SetupFlags(f *pflag.FlagSet) {
 	f.String("ui.collapseGroups", "collapsedOnMobile", "Default state for alert groups")
 }
 
-func readConfigFile(k *koanf.Koanf, flags *pflag.FlagSet) string {
+func readConfigFile(k *koanf.Koanf, flags *pflag.FlagSet) (string, error) {
 	var configFile string
 
 	// 1. Load file from flags is set
@@ -141,11 +142,11 @@ func readConfigFile(k *koanf.Koanf, flags *pflag.FlagSet) string {
 	}
 	if configFile != "" {
 		if err := k.Load(file.Provider(configFile), yamlParser.Parser()); err != nil {
-			log.Fatalf("Failed to load configuration file %q: %v", configFile, err)
+			return "", fmt.Errorf("Failed to load configuration file %q: %v", configFile, err)
 		}
-		return configFile
+		return configFile, nil
 	}
-	return configFile
+	return configFile, nil
 }
 
 func readEnvVariables(k *koanf.Koanf) {
@@ -206,14 +207,18 @@ func readFlags(k *koanf.Koanf, flags *pflag.FlagSet) {
 // 1. CLI flags
 // 2. Config file
 // 3. Environment variables
-func (config *configSchema) Read(flags *pflag.FlagSet) string {
+func (config *configSchema) Read(flags *pflag.FlagSet) (string, error) {
 	k := koanf.New(".")
 	var configFileUsed string
 
 	// 3. read all environemnt variables
 	readEnvVariables(k)
 	// 2. read config file
-	if cf := readConfigFile(k, flags); cf != "" {
+	cf, err := readConfigFile(k, flags)
+	if err != nil {
+		return "", err
+	}
+	if cf != "" {
 		configFileUsed = cf
 	}
 	// 1. read flags
@@ -233,30 +238,30 @@ func (config *configSchema) Read(flags *pflag.FlagSet) string {
 		FlatPaths:     false,
 		DecoderConfig: &dConf,
 	}
-	err := k.UnmarshalWithConf("", &config, kConf)
+	err = k.UnmarshalWithConf("", &config, kConf)
 	if err != nil {
-		log.Fatalf("Failed to unmarshal configuration: %v", err)
+		return "", fmt.Errorf("Failed to unmarshal configuration: %v", err)
 	}
 
 	if config.Authentication.Header.Name != "" && len(config.Authentication.BasicAuth.Users) > 0 {
-		log.Fatalf("Both authentication.basicAuth.users and authentication.header.name is set, only one can be enabled")
+		return "", fmt.Errorf("Both authentication.basicAuth.users and authentication.header.name is set, only one can be enabled")
 	}
 
 	if config.Authentication.Header.ValueRegex != "" {
 		_, err = regex.CompileAnchored(config.Authentication.Header.ValueRegex)
 		if err != nil {
-			log.Fatalf("Invalid regex for authentication.header.value_re: %s", err.Error())
+			return "", fmt.Errorf("Invalid regex for authentication.header.value_re: %s", err.Error())
 		}
 		if config.Authentication.Header.Name == "" {
-			log.Fatalf("authentication.header.name is required when authentication.header.value_re is set")
+			return "", fmt.Errorf("authentication.header.name is required when authentication.header.value_re is set")
 		}
 	} else if config.Authentication.Header.Name != "" {
-		log.Fatalf("authentication.header.value_re is required when authentication.header.name is set")
+		return "", fmt.Errorf("authentication.header.value_re is required when authentication.header.name is set")
 	}
 
 	for _, u := range config.Authentication.BasicAuth.Users {
 		if u.Username == "" || u.Password == "" {
-			log.Fatalf("authentication.basicAuth.users require both username and password to be set")
+			return "", fmt.Errorf("authentication.basicAuth.users require both username and password to be set")
 		}
 	}
 
@@ -265,7 +270,7 @@ func (config *configSchema) Read(flags *pflag.FlagSet) string {
 	}
 
 	if !slices.StringInSlice([]string{"omit", "include", "same-origin"}, config.Alertmanager.CORS.Credentials) {
-		log.Fatalf("Invalid alertmanager.cors.credentials value '%s', allowed options: omit, inclue, same-origin", config.Alertmanager.CORS.Credentials)
+		return "", fmt.Errorf("Invalid alertmanager.cors.credentials value '%s', allowed options: omit, inclue, same-origin", config.Alertmanager.CORS.Credentials)
 	}
 
 	for i, s := range config.Alertmanager.Servers {
@@ -279,47 +284,47 @@ func (config *configSchema) Read(flags *pflag.FlagSet) string {
 			config.Alertmanager.Servers[i].CORS.Credentials = config.Alertmanager.CORS.Credentials
 		}
 		if !slices.StringInSlice([]string{"omit", "include", "same-origin"}, config.Alertmanager.Servers[i].CORS.Credentials) {
-			log.Fatalf("Invalid cors.credentials value '%s' for alertmanager '%s', allowed options: omit, inclue, same-origin", config.Alertmanager.Servers[i].CORS.Credentials, s.Name)
+			return "", fmt.Errorf("Invalid cors.credentials value '%s' for alertmanager '%s', allowed options: omit, inclue, same-origin", config.Alertmanager.Servers[i].CORS.Credentials, s.Name)
 		}
 	}
 
 	for _, authGroup := range config.Authorization.Groups {
 		if authGroup.Name == "" {
-			log.Fatalf("'name' is required for every authorization group")
+			return "", fmt.Errorf("'name' is required for every authorization group")
 		}
 		if len(authGroup.Members) == 0 {
-			log.Fatalf("'members' is required for every authorization group")
+			return "", fmt.Errorf("'members' is required for every authorization group")
 		}
 	}
 
 	for labelName, customColors := range config.Labels.Color.Custom {
 		for i, customColor := range customColors {
 			if customColor.Value == "" && customColor.ValueRegex == "" {
-				log.Fatalf("Custom label color for '%s' is missing 'value' or 'value_re'", labelName)
+				return "", fmt.Errorf("Custom label color for '%s' is missing 'value' or 'value_re'", labelName)
 			}
 			if customColor.ValueRegex != "" {
 				config.Labels.Color.Custom[labelName][i].CompiledRegex, err = regex.CompileAnchored(customColor.ValueRegex)
 				if err != nil {
-					log.Fatalf("Failed to parse custom color regex rule '%s' for '%s' label: %s", customColor.ValueRegex, labelName, err)
+					return "", fmt.Errorf("Failed to parse custom color regex rule '%s' for '%s' label: %s", customColor.ValueRegex, labelName, err)
 				}
 			}
 		}
 	}
 
 	if !slices.StringInSlice([]string{"disabled", "startsAt", "label"}, config.Grid.Sorting.Order) {
-		log.Fatalf("Invalid grid.sorting.order value '%s', allowed options: disabled, startsAt, label", config.Grid.Sorting.Order)
+		return "", fmt.Errorf("Invalid grid.sorting.order value '%s', allowed options: disabled, startsAt, label", config.Grid.Sorting.Order)
 	}
 
 	if !slices.StringInSlice([]string{"expanded", "collapsed", "collapsedOnMobile"}, config.UI.CollapseGroups) {
-		log.Fatalf("Invalid ui.collapseGroups value '%s', allowed options: expanded, collapsed, collapsedOnMobile", config.UI.CollapseGroups)
+		return "", fmt.Errorf("Invalid ui.collapseGroups value '%s', allowed options: expanded, collapsed, collapsedOnMobile", config.UI.CollapseGroups)
 	}
 
 	if !slices.StringInSlice([]string{"light", "dark", "auto"}, config.UI.Theme) {
-		log.Fatalf("Invalid ui.theme value '%s', allowed options: light, dark, auto", config.UI.Theme)
+		return "", fmt.Errorf("Invalid ui.theme value '%s', allowed options: light, dark, auto", config.UI.Theme)
 	}
 
 	if config.Listen.Prefix != "" && !strings.HasPrefix(config.Listen.Prefix, "/") {
-		log.Fatalf("listen.prefix must start with '/', got %q", config.Listen.Prefix)
+		return "", fmt.Errorf("listen.prefix must start with '/', got %q", config.Listen.Prefix)
 	}
 
 	// accept single Alertmanager server from flag/env if nothing is set yet
@@ -340,7 +345,7 @@ func (config *configSchema) Read(flags *pflag.FlagSet) string {
 
 	Config = config
 
-	return configFileUsed
+	return configFileUsed, nil
 }
 
 // LogValues will dump runtime config to logs

internal/config/config_test.go

@@ -150,10 +150,10 @@ ui:
 	}
 }
 
-func mockConfigRead() {
+func mockConfigRead() (string, error) {
 	f := pflag.NewFlagSet(".", pflag.ExitOnError)
 	SetupFlags(f)
-	Config.Read(f)
+	return Config.Read(f)
 }
 
 func TestReadConfig(t *testing.T) {
@@ -177,7 +177,7 @@ func TestReadConfig(t *testing.T) {
 	os.Setenv("LISTEN_PORT", "80")
 	os.Setenv("SENTRY_PRIVATE", "secret key")
 	os.Setenv("SENTRY_PUBLIC", "public key")
-	mockConfigRead()
+	_, _ = mockConfigRead()
 	testReadConfig(t)
 }
 
@@ -190,7 +190,7 @@ func TestReadSimpleConfig(t *testing.T) {
 	os.Setenv("ALERTMANAGER_TIMEOUT", "15s")
 	os.Setenv("ALERTMANAGER_PROXY", "true")
 	os.Setenv("ALERTMANAGER_INTERVAL", "3m")
-	mockConfigRead()
+	_, _ = mockConfigRead()
 	if len(Config.Alertmanager.Servers) != 1 {
 		t.Errorf("Expected 1 Alertmanager server, got %d", len(Config.Alertmanager.Servers))
 	} else {
@@ -255,7 +255,7 @@ func TestUrlSecretTest(t *testing.T) {
 
 // FIXME check logged values
 func TestLogValues(t *testing.T) {
-	mockConfigRead()
+	_, _ = mockConfigRead()
 	Config.LogValues()
 }
 
@@ -263,15 +263,9 @@ func TestInvalidGridSortingOrder(t *testing.T) {
 	resetEnv()
 	os.Setenv("GRID_SORTING_ORDER", "foo")
 
-	log.SetLevel(log.PanicLevel)
-	defer func() { log.StandardLogger().ExitFunc = nil }()
-	var wasFatal bool
-	log.StandardLogger().ExitFunc = func(int) { wasFatal = true }
-
-	mockConfigRead()
-
-	if !wasFatal {
-		t.Error("Invalid grid.sorting.order value didn't cause log.Fatal()")
+	_, err := mockConfigRead()
+	if err == nil {
+		t.Error("Invalid grid.sorting.order value didn't return any error")
 	}
 }
 
@@ -279,15 +273,9 @@ func TestInvalidUICollapseGroups(t *testing.T) {
 	resetEnv()
 	os.Setenv("UI_COLLAPSEGROUPS", "foo")
 
-	log.SetLevel(log.PanicLevel)
-	defer func() { log.StandardLogger().ExitFunc = nil }()
-	var wasFatal bool
-	log.StandardLogger().ExitFunc = func(int) { wasFatal = true }
-
-	mockConfigRead()
-
-	if !wasFatal {
-		t.Error("Invalid ui.collapseGroups value didn't cause log.Fatal()")
+	_, err := mockConfigRead()
+	if err == nil {
+		t.Error("Invalid ui.collapseGroups value didn't return any error")
 	}
 }
 
@@ -295,15 +283,9 @@ func TestInvalidUITheme(t *testing.T) {
 	resetEnv()
 	os.Setenv("UI_THEME", "foo")
 
-	log.SetLevel(log.PanicLevel)
-	defer func() { log.StandardLogger().ExitFunc = nil }()
-	var wasFatal bool
-	log.StandardLogger().ExitFunc = func(int) { wasFatal = true }
-
-	mockConfigRead()
-
-	if !wasFatal {
-		t.Error("Invalid ui.theme value didn't cause log.Fatal()")
+	_, err := mockConfigRead()
+	if err == nil {
+		t.Error("Invalid ui.theme value didn't return any error")
 	}
 }
 
@@ -311,22 +293,16 @@ func TestInvalidCORSCredentials(t *testing.T) {
 	resetEnv()
 	os.Setenv("ALERTMANAGER_CORS_CREDENTIALS", "foo")
 
-	log.SetLevel(log.PanicLevel)
-	defer func() { log.StandardLogger().ExitFunc = nil }()
-	var wasFatal bool
-	log.StandardLogger().ExitFunc = func(int) { wasFatal = true }
-
-	mockConfigRead()
-
-	if !wasFatal {
-		t.Error("Invalid alertmanager.cors.credentials value didn't cause log.Fatal()")
+	_, err := mockConfigRead()
+	if err == nil {
+		t.Error("Invalid alertmanager.cors.credentials value didn't return any error")
 	}
 }
 
 func TestDefaultConfig(t *testing.T) {
 	resetEnv()
 	log.SetLevel(log.ErrorLevel)
-	mockConfigRead()
+	_, _ = mockConfigRead()
 
 	expectedConfig := configSchema{}
 	expectedConfig.Annotations.Hidden = []string{}

scripts/testscript-one-by-one.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -e
+
+for I in ./cmd/karma/tests/testscript/*.txt ; do
+    T=`basename "${I}" | cut -d. -f1`
+    echo ">>> ${T}"
+    go test -count=1 -timeout=30s -v -run=TestScript/${T} ./cmd/karma
+done