fix(ci): tweak fragile tests

cmd/karma/main.go

@@ -486,10 +486,11 @@ func serve(errorHandling pflag.ErrorHandling) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 	if err := httpServer.Shutdown(ctx); err != nil {
+		_ = removePidFile()
 		return fmt.Errorf("shutdown error: %s", err)
 	}
-	log.Info().Msg("HTTP server shut down")
 
+	log.Info().Msg("HTTP server shut down")
 	return removePidFile()
 }
 

cmd/karma/tests/testscript/035_log_requests.txt

@@ -1,14 +1,12 @@
 # Works in simple mode when single --alertmanager.uri flag is passed
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8035 --log.requests
 ! stdout .
 stderr 'msg="Request completed" address=127.0.0.1:[0-9]+ bytes=[0-9]+ code=200 duration=".+" method=GET path=/alerts.json'
 stderr 'msg="Request completed" address=127.0.0.1:[0-9]+ bytes=[0-9]+ code=200 duration=".+" method=GET path=/alerts.json'
 
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
 curl -so /dev/null http://127.0.0.1:8035/alerts.json

cmd/karma/tests/testscript/068_sentry.txt

@@ -1,6 +1,6 @@
 # Configures sentry when enabled
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --log.config=true --config.file=karma.yaml --listen.address=127.0.0.1 --listen.port=8068
 ! stdout .
 cmp stderr stderr.txt
@@ -138,8 +138,6 @@ sentry:
   public: https://examplePublicKey@o0.ingest.sentry.io/1
 
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
 cat karma.pid | xargs kill

cmd/karma/tests/testscript/069_simple_config.txt

@@ -1,6 +1,6 @@
 # Works in simple mode when single --alertmanager.uri flag is passed
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8069
 ! stdout .
 cmp stderr stderr.txt
@@ -22,8 +22,6 @@ level=info msg="Shutting down HTTP server"
 level=info msg="HTTP server shut down"
 level=info msg="Removing PID file" path=karma.pid
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
 cat karma.pid | xargs kill

cmd/karma/tests/testscript/073_pid_file_remove_error.txt

@@ -1,6 +1,6 @@
 # Fails is we cannot remove a PID file
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-fail --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8073
 ! stdout .
 cmp stderr stderr.txt
@@ -23,8 +23,6 @@ level=info msg="HTTP server shut down"
 level=info msg="Removing PID file" path=karma.pid
 level=error msg="Execution failed" error="failed to remove PID file: remove karma.pid: no such file or directory"
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
 PID=$(cat karma.pid)

cmd/karma/tests/testscript/076_basicAuth_uri.txt

@@ -1,14 +1,12 @@
 # Basic Auth headers are passed to the UI in the API response
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://foo:bar@127.0.0.1 --listen.address=127.0.0.1 --listen.port=8076
 wait
 stdout '"headers":{"Authorization":"Basic Zm9vOmJhcg=="}'
 stdout '"publicURI":"http://foo:bar@127.0.0.1"'
 
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
 curl -s http://127.0.0.1:8076/alerts.json

cmd/karma/tests/testscript/083_metrics.txt

@@ -1,6 +1,6 @@
 # GET /metrics returns same list of metrics
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8083
 ! stdout .
 cmp stderr stderr.txt
@@ -129,8 +129,6 @@ level=info msg="Shutting down HTTP server"
 level=info msg="HTTP server shut down"
 level=info msg="Removing PID file" path=karma.pid
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 5
 curl -s http://localhost:8083/metrics | grep -vE 'process_|go_info' | perl -pe "s/^([a-zA-Z].+)[ ]([0-9\.\-\+eE]+)$/\1/g" > curl.txt

cmd/karma/tests/testscript/085_debug.txt

@@ -1,6 +1,6 @@
 # Enables pprof endpoints when debug is enabled
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8085 --debug
 ! stdout .
 cmp stderr stderr.txt
@@ -22,8 +22,6 @@ level=info msg="Shutting down HTTP server"
 level=info msg="HTTP server shut down"
 level=info msg="Removing PID file" path=karma.pid
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
 curl -s -o heap.gz http://localhost:8085/debug/pprof/heap

cmd/karma/tests/testscript/087_listen_tls.txt

@@ -1,7 +1,7 @@
 # Listens on HTTPS when enabled
 
-exec sh -x ./tls.sh
-exec sh -x ./test.sh &
+exec bash -x ./tls.sh
+exec bash -x ./test.sh &
 karma.bin-should-work --pid-file=karma.pid --config.file=karma.yaml --listen.address=127.0.0.1 --listen.port=8087
 ! stdout .
 wait
@@ -48,8 +48,6 @@ openssl req -new -key server.key -out server.csr -subj "/C=CI/ST=CI/L=CI/O=CI/CN
 openssl x509 -req -in server.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out server.pem -days 7
 
 -- test.sh --
-#!/bin/sh
-
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
 curl -s --resolve karma.example.com:8087:127.0.0.1 --cacert ca.pem -o http1.txt --http1.1 https://karma.example.com:8087/health

cmd/karma/tests/testscript/094_shutdown_slow_client.txt

@@ -1,6 +1,6 @@
 # Terminates with an error on slow client
 
-exec sh -x ./test.sh &
+exec bash -x ./test.sh &
 karma.bin-should-fail --pid-file=karma.pid --alertmanager.uri http://127.0.0.1 --listen.address=127.0.0.1 --listen.port 8094
 ! stdout .
 cmp stderr stderr.txt
@@ -21,6 +21,7 @@ level=info msg="Collection completed"
 level=info msg="Done, starting HTTP server"
 level=info msg="Starting HTTP server" address=127.0.0.1:8094
 level=info msg="Shutting down HTTP server"
+level=info msg="Removing PID file" path=karma.pid
 level=error msg="Execution failed" error="shutdown error: context deadline exceeded"
 -- karma.yaml --
 alertmanager:
@@ -30,10 +31,12 @@ alertmanager:
       proxy: true
 debug: true
 -- test.sh --
-#!/bin/sh
+trap "echo got signal" SIGTERM SIGINT
 
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 1
-curl -s --limit-rate 1 http://127.0.0.1:8094/debug/pprof/profile?seconds=45 &
-sleep 1
+(curl -s --limit-rate 1 http://127.0.0.1:8094/debug/pprof/profile?seconds=35 || echo "curl completed") &
+sleep 3
 cat karma.pid | xargs kill
+while [ -f karma.pid ]; do sleep 1 ; done
+killall -v curl || echo "no curl running"

cmd/karma/tests/testscript/096_tls_client.txt

@@ -1,8 +1,8 @@
 # Correctly passes TLS client config to the upstream
 
-exec sh -x ./tls.sh
-exec sh -x ./test.sh &
-exec sh -c 'I=0 ; while [ ! -f alertmanager.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done'
+exec bash -x ./tls.sh
+exec bash -x ./test.sh &
+exec bash -c 'I=0 ; while [ ! -f alertmanager.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done'
 karma.bin-should-work --pid-file=karma.pid
 ! stdout .
 stderr 'level=info msg="Upstream version" alertmanager=client-auth version=0.21.0'
@@ -60,8 +60,6 @@ openssl x509 -req -days 7 -extfile karma.conf -extensions SAN -in karma.csr -CA
 openssl x509 -in karma.pem -text
 
 -- test.sh --
-#!/bin/sh
-
 env GOCACHE=$TMPDIR go run alertmanager.go &
 
 I=0
@@ -76,7 +74,7 @@ alertmanager:
   interval: 1h
   servers:
     - name: client-auth
-      uri: https://127.0.0.1:9084
+      uri: https://127.0.0.1:9096
       timeout: 10s
       tls:
         ca: ca.pem
@@ -84,7 +82,7 @@ alertmanager:
         key: karma.key
 listen:
   address: 127.0.0.1
-  port: 8084
+  port: 8096
 
 -- alertmanager.go --
 package main
@@ -161,13 +159,13 @@ func main() {
 		InsecureSkipVerify: true,
 	}
 
-	listener, err := net.Listen("tcp", "127.0.0.1:9084")
+	listener, err := net.Listen("tcp", "127.0.0.1:9096")
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	server := &http.Server{
-		Addr:      "127.0.0.1:9084",
+		Addr:      "127.0.0.1:9096",
 		TLSConfig: tlsConfig,
 	}
 

scripts/test-unit.sh

@@ -17,7 +17,7 @@ I=0
 for d in $PKGS; do
     I=$((I+1))
     COVFILE="profile.test.${I}"
-    (go test -coverprofile="${COVFILE}" -coverpkg="$COVERPKG" $d 2>&1 || exit 2) \
+    (go test -count=1 -coverprofile="${COVFILE}" -coverpkg="$COVERPKG" $d 2>&1 || exit 2) \
         | grep -v 'warning: no packages being tested depend on matches for pattern' \
         | sed s/'of statements in .*'/''/g
 done