feat(tests): add basic testscript tests

cmd/karma/main.go

@@ -117,7 +117,7 @@ func setupMetrics(router *gin.Engine) {
 	router.GET(getViewURL("/metrics"), promHandler(promhttp.Handler()))
 }
 
-func setupUpstreams() {
+func setupUpstreams() error {
 	for _, s := range config.Config.Alertmanager.Servers {
 
 		var httpTransport http.RoundTripper
@@ -126,7 +126,7 @@ func setupUpstreams() {
 		if s.TLS.CA != "" || s.TLS.Cert != "" || s.TLS.InsecureSkipVerify {
 			httpTransport, err = alertmanager.NewHTTPTransport(s.TLS.CA, s.TLS.Cert, s.TLS.Key, s.TLS.InsecureSkipVerify)
 			if err != nil {
-				log.Fatalf("Failed to create HTTP transport for Alertmanager '%s' with URI '%s': %s", s.Name, uri.SanitizeURI(s.URI), err)
+				return fmt.Errorf("Failed to create HTTP transport for Alertmanager '%s' with URI '%s': %s", s.Name, uri.SanitizeURI(s.URI), err)
 			}
 		}
 
@@ -140,16 +140,18 @@ func setupUpstreams() {
 			alertmanager.WithHTTPHeaders(s.Headers),
 		)
 		if err != nil {
-			log.Fatalf("Failed to create Alertmanager '%s' with URI '%s': %s", s.Name, uri.SanitizeURI(s.URI), err)
+			return fmt.Errorf("Failed to create Alertmanager '%s' with URI '%s': %s", s.Name, uri.SanitizeURI(s.URI), err)
 		}
 		err = alertmanager.RegisterAlertmanager(am)
 		if err != nil {
-			log.Fatalf("Failed to register Alertmanager '%s' with URI '%s': %s", s.Name, uri.SanitizeURI(s.URI), err)
+			return fmt.Errorf("Failed to register Alertmanager '%s' with URI '%s': %s", s.Name, uri.SanitizeURI(s.URI), err)
 		}
 	}
+
+	return nil
 }
 
-func setupLogger() {
+func setupLogger() error {
 	switch config.Config.Log.Level {
 	case "debug":
 		log.SetLevel(log.DebugLevel)
@@ -164,7 +166,7 @@ func setupLogger() {
 	case "panic":
 		log.SetLevel(log.PanicLevel)
 	default:
-		log.Fatalf("Unknown log level '%s'", config.Config.Log.Level)
+		return fmt.Errorf("Unknown log level '%s'", config.Config.Log.Level)
 	}
 
 	switch config.Config.Log.Format {
@@ -173,26 +175,31 @@ func setupLogger() {
 	case "json":
 		log.SetFormatter(&log.JSONFormatter{})
 	default:
-		log.Fatalf("Unknown log format '%s'", config.Config.Log.Format)
+		return fmt.Errorf("Unknown log format '%s'", config.Config.Log.Format)
 	}
+
+	return nil
 }
 
-func main() {
+func mainSetup() (*gin.Engine, error) {
 	printVersion := pflag.Bool("version", false, "Print version and exit")
 	validateConfig := pflag.Bool("check-config", false, "Validate configuration and exit")
 	pflag.Parse()
 
 	if *printVersion {
 		fmt.Println(version)
-		return
+		return nil, nil
 	}
 
 	config.Config.Read()
-	setupLogger()
+	err := setupLogger()
+	if err != nil {
+		return nil, err
+	}
 
 	// timer duration cannot be zero second or a negative one
 	if config.Config.Alertmanager.Interval <= time.Second*0 {
-		log.Fatalf("Invalid AlertmanagerTTL value '%v'", config.Config.Alertmanager.Interval)
+		return nil, fmt.Errorf("Invalid AlertmanagerTTL value '%v'", config.Config.Alertmanager.Interval)
 	}
 
 	log.Infof("Version: %s", version)
@@ -203,11 +210,11 @@ func main() {
 	linkDetectRules := []models.LinkDetectRule{}
 	for _, rule := range config.Config.Silences.Comments.LinkDetect.Rules {
 		if rule.Regex == "" || rule.URITemplate == "" {
-			log.Fatalf("Invalid link detect rule, regex '%s' uriTemplate '%s'", rule.Regex, rule.URITemplate)
+			return nil, fmt.Errorf("Invalid link detect rule, regex '%s' uriTemplate '%s'", rule.Regex, rule.URITemplate)
 		}
 		re, err := regexp.Compile(rule.Regex)
 		if err != nil {
-			log.Fatalf("Invalid link detect rule '%s': %s", rule.Regex, err)
+			return nil, fmt.Errorf("Invalid link detect rule '%s': %s", rule.Regex, err)
 		}
 		linkDetectRules = append(linkDetectRules, models.LinkDetectRule{Regex: re, URITemplate: rule.URITemplate})
 	}
@@ -215,26 +222,20 @@ func main() {
 
 	apiCache = cache.New(cache.NoExpiration, 10*time.Second)
 
-	setupUpstreams()
+	err = setupUpstreams()
+	if err != nil {
+		return nil, err
+	}
 
 	if len(alertmanager.GetAlertmanagers()) == 0 {
-		log.Fatal("No valid Alertmanager URIs defined")
+		return nil, fmt.Errorf("No valid Alertmanager URIs defined")
 	}
 
 	if *validateConfig {
 		log.Info("Configuration is valid")
-		return
+		return nil, nil
 	}
 
-	// before we start try to fetch data from Alertmanager
-	log.Info("Initial Alertmanager query")
-	pullFromAlertmanager()
-	log.Info("Done, starting HTTP server")
-
-	// background loop that will fetch updates from Alertmanager
-	ticker = time.NewTicker(config.Config.Alertmanager.Interval)
-	go Tick()
-
 	switch config.Config.Debug {
 	case true:
 		gin.SetMode(gin.DebugMode)
@@ -263,10 +264,31 @@ func main() {
 	for _, am := range alertmanager.GetAlertmanagers() {
 		err := setupRouterProxyHandlers(router, am)
 		if err != nil {
-			log.Fatalf("Failed to setup proxy handlers for Alertmanager '%s': %s", am.Name, err)
+			return nil, fmt.Errorf("Failed to setup proxy handlers for Alertmanager '%s': %s", am.Name, err)
 		}
 	}
 
+	return router, nil
+}
+
+func main() {
+	router, err := mainSetup()
+	if err != nil {
+		log.Fatal(err)
+	}
+	if router == nil {
+		return
+	}
+
+	// before we start try to fetch data from Alertmanager
+	log.Info("Initial Alertmanager query")
+	pullFromAlertmanager()
+	log.Info("Done, starting HTTP server")
+
+	// background loop that will fetch updates from Alertmanager
+	ticker = time.NewTicker(config.Config.Alertmanager.Interval)
+	go Tick()
+
 	listen := fmt.Sprintf("%s:%d", config.Config.Listen.Address, config.Config.Listen.Port)
 	httpServer := &http.Server{
 		Addr:    listen,

cmd/karma/main_test.go

@@ -23,7 +23,10 @@ func TestLogConfig(t *testing.T) {
 
 	for val, level := range logLevels {
 		config.Config.Log.Level = val
-		setupLogger()
+		err := setupLogger()
+		if err != nil {
+			t.Error(err)
+		}
 		if log.GetLevel() != level {
 			t.Errorf("Config.Log.Level=%s resulted in invalid log level %s", val, log.GetLevel())
 		}

cmd/karma/script_test.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/rogpeppe/go-internal/testscript"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func mainShoulFail() int {
+	_, err := mainSetup()
+	if err != nil {
+		log.Error(err)
+	} else {
+		log.Error("No error logged")
+		return 100
+	}
+	return 0
+}
+
+func mainShouldWork() int {
+	_, err := mainSetup()
+	if err != nil {
+		log.Error(err)
+		return 100
+	}
+	return 0
+}
+
+func TestMain(m *testing.M) {
+	os.Exit(testscript.RunMain(m, map[string]func() int{
+		"karma.bin-should-fail": mainShoulFail,
+		"karma.bin-should-work": mainShouldWork,
+	}))
+}
+
+func TestScripts(t *testing.T) {
+	testscript.Run(t, testscript.Params{
+		Dir: "testdata",
+	})
+}

cmd/karma/testdata/check-config.txt

@@ -0,0 +1,5 @@
+# Validates config when --check-config is passed
+karma.bin-should-work --log.format=text --log.config=false --check-config --alertmanager.uri=http://localhost
+! stdout .
+stderr 'msg="Configuration is valid"'
+! stderr 'level=error'

cmd/karma/testdata/duplicated_alertmanager_name.txt

@@ -0,0 +1,12 @@
+# Raises an error if we have 2 instances with the same name
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --config.file=karma.yaml
+! stdout .
+stderr 'msg="Failed to register Alertmanager ''am1'' with URI ''https://localhost:9094'': alertmanager upstream ''am1'' already exist"'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: am1
+      uri: https://localhost:9093
+    - name: am1
+      uri: https://localhost:9094

cmd/karma/testdata/duplicated_alertmanager_uri.txt

@@ -0,0 +1,12 @@
+# Raises an error if we have 2 instances with the same URI
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --config.file karma.yaml
+! stdout .
+stderr 'msg="Failed to register Alertmanager ''am2'' with URI ''https://localhost:9093'': alertmanager upstream ''am1'' already collects from ''https://localhost:9093''"'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: am1
+      uri: https://localhost:9093
+    - name: am2
+      uri: https://localhost:9093

cmd/karma/testdata/invalid_log_format.txt

@@ -0,0 +1,4 @@
+# Raises an error if invalid log format is passed
+karma.bin-should-fail --log.format=text --log.config=false --log.format=xml
+! stdout .
+stderr 'msg="Unknown log format ''xml''"'

cmd/karma/testdata/invalid_log_level.txt

@@ -0,0 +1,4 @@
+# Raises an error if invalid log level is passed
+karma.bin-should-fail --log.format=text --log.config=false --log.level=foobar
+! stdout .
+stderr 'msg="Unknown log level ''foobar''"'

cmd/karma/testdata/invalid_tls_config.txt

@@ -0,0 +1,15 @@
+# Raises an error if tls config is invalid
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --config.file=karma.yaml
+! stdout .
+stderr 'msg="Failed to create HTTP transport for Alertmanager ''client-auth'' with URI ''https://localhost:9093'': open /xxx/xxx-ca-bundle.crt: no such file or directory"'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: client-auth
+      uri: https://localhost:9093
+      timeout: 10s
+      tls:
+        ca: /xxx/xxx-ca-bundle.crt
+        cert: /xxx/karma/client.pem
+        key: /xxx/karma/client.key

cmd/karma/testdata/invalid_ttl.txt

@@ -0,0 +1,4 @@
+# Raises an error if negative refresh interval is passed
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --alertmanager.interval=-4s
+! stdout .
+stderr 'msg="Invalid AlertmanagerTTL value ''-4s''"'

cmd/karma/testdata/invalid_uri.txt

@@ -0,0 +1,4 @@
+# Raises an error if alertmanager URI is invalid
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --alertmanager.uri httpz://username:secret@localhost
+! stdout .
+stderr 'msg="Failed to create Alertmanager ''default'' with URI ''httpz://username:xxx@localhost'': unsupported URI scheme ''httpz'' in ''httpz://username:xxx@localhost''"'

cmd/karma/testdata/linkDetect_invalid_regex.txt

@@ -0,0 +1,16 @@
+# Raises an error if linkDetect config is missing regex rule
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --config.file=karma.yaml
+! stdout .
+stderr 'msg="Invalid link detect rule ''foo\+\+\+\+\+\+'': error parsing regexp: invalid nested repetition operator: `\+\+`"'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: https://localhost:9093
+silences:
+  comments:
+    linkDetect:
+      rules:
+        - regex: "foo++++++"
+          uriTemplate: https://jira.example.com/

cmd/karma/testdata/linkDetect_no_regex.txt

@@ -0,0 +1,15 @@
+# Raises an error if linkDetect config is missing regex rule
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --config.file=karma.yaml
+! stdout .
+stderr 'msg="Invalid link detect rule, regex '''' uriTemplate ''https://jira.example.com/''"'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: https://localhost:9093
+silences:
+  comments:
+    linkDetect:
+      rules:
+        - uriTemplate: https://jira.example.com/

cmd/karma/testdata/linkDetect_no_uri.txt

@@ -0,0 +1,15 @@
+# Raises an error if linkDetect config is missing uriTemplate
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error --config.file=karma.yaml
+! stdout .
+stderr 'msg="Invalid link detect rule, regex ''DEVOPS-\[0-9\]\+'' uriTemplate ''''"'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: https://localhost:9093
+silences:
+  comments:
+    linkDetect:
+      rules:
+        - regex: "DEVOPS-[0-9]+"

cmd/karma/testdata/linkDetect_valid_config.txt

@@ -0,0 +1,17 @@
+# Config is valid with correct linkDetect rules
+karma.bin-should-work --log.format=text --log.config=false --config.file=karma.yaml --check-config
+! stdout .
+stderr 'msg="Configuration is valid"'
+! stderr 'level=error'
+
+-- karma.yaml --
+alertmanager:
+  servers:
+    - name: default
+      uri: https://localhost:9093
+silences:
+  comments:
+    linkDetect:
+      rules:
+        - regex: "(DEVOPS-[0-9]+)"
+          uriTemplate: https://jira.example.com/browse/$1

cmd/karma/testdata/log_format_json.txt

@@ -0,0 +1,4 @@
+# Logs messages as JSON when log.format=json is passed
+karma.bin-should-fail --log.format=json --log.level=error
+! stdout .
+stderr '^{"level":"error","msg":"No valid Alertmanager URIs defined"'

cmd/karma/testdata/no_args.txt

@@ -0,0 +1,4 @@
+# Raises an error if no alertmanager uri is set
+karma.bin-should-fail --log.format=text --log.config=false --log.level=error
+! stdout .
+stderr 'msg="No valid Alertmanager URIs defined"'

cmd/karma/testdata/version.txt

@@ -0,0 +1,4 @@
+# Prints version when --version is passed
+karma.bin-should-work --version
+stdout 'dev\n'
+! stderr .

cmd/karma/views_test.go

@@ -33,7 +33,10 @@ func mockConfig() {
 	config.Config.Read()
 	if !upstreamSetup {
 		upstreamSetup = true
-		setupUpstreams()
+		err := setupUpstreams()
+		if err != nil {
+			log.Fatal(err)
+		}
 	}
 }