diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a60952f4f..834af8208 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -44,11 +44,7 @@ jobs:
         run: make mock-assets
 
       - name: Test Go code
-        uses: nick-invision/retry@v2.8.1
-        with:
-          timeout_minutes: 10
-          max_attempts: 5
-          command: make test-go
+        run: make test-go
 
       - name: Report code coverage
         uses: codecov/codecov-action@v3.1.0
diff --git a/cmd/karma/tests/testscript/099_alert_history.txt b/cmd/karma/tests/testscript/110_alert_history.txt
similarity index 91%
rename from cmd/karma/tests/testscript/099_alert_history.txt
rename to cmd/karma/tests/testscript/110_alert_history.txt
index 8e01c440a..776095c02 100644
--- a/cmd/karma/tests/testscript/099_alert_history.txt
+++ b/cmd/karma/tests/testscript/110_alert_history.txt
@@ -1,7 +1,7 @@
 # GET /history.json
 
 exec bash -x ./test.sh &
-karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8099 --history.enabled=true --history.timeout=10s
+karma.bin-should-work --pid-file=karma.pid --alertmanager.uri=http://127.0.0.1 --listen.address=127.0.0.1 --listen.port=8110 --history.enabled=true --history.timeout=10s
 ! stdout .
 cmp stderr stderr.txt
 
@@ -20,15 +20,15 @@ level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial t
 level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
 level=info msg="Collection completed"
 level=info msg="Done, starting HTTP server"
-level=info msg="Starting HTTP server" address=127.0.0.1:8099
+level=info msg="Starting HTTP server" address=127.0.0.1:8110
 level=info msg="Shutting down HTTP server"
 level=info msg="HTTP server shut down"
 level=info msg="Removing PID file" path=karma.pid
 -- query.json --
 {
     "sources": [
-        "http://127.0.0.1:9099",
-        "http://127.0.0.1:9099"
+        "http://127.0.0.1:9110",
+        "http://127.0.0.1:9110"
     ],
     "labels": {
         "alertname": "Fake Alert"
@@ -39,7 +39,7 @@ env GOCACHE=$TMPDIR go run prometheus.go &
 while [ ! -f prometheus.pid ]; do sleep 1 ; done
 while [ ! -f karma.pid ]; do sleep 1 ; done
 sleep 5
-curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8099/history.json
+curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8110/history.json
 cat karma.pid | xargs kill
 cat prometheus.pid | xargs kill
 
@@ -93,13 +93,13 @@ func main() {
 	http.HandleFunc("/api/v1/labels", labelNames)
 	http.HandleFunc("/api/v1/query_range", query)
 
-	listener, err := net.Listen("tcp", "127.0.0.1:9099")
+	listener, err := net.Listen("tcp", "127.0.0.1:9110")
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	server := &http.Server{
-		Addr: "127.0.0.1:9099",
+		Addr: "127.0.0.1:9110",
 	}
 
 	go func() {
diff --git a/cmd/karma/tests/testscript/104_alert_history_tls_bad.txt b/cmd/karma/tests/testscript/111_alert_history_tls_bad.txt
similarity index 91%
rename from cmd/karma/tests/testscript/104_alert_history_tls_bad.txt
rename to cmd/karma/tests/testscript/111_alert_history_tls_bad.txt
index 95f7ccc40..957ad2866 100644
--- a/cmd/karma/tests/testscript/104_alert_history_tls_bad.txt
+++ b/cmd/karma/tests/testscript/111_alert_history_tls_bad.txt
@@ -21,8 +21,8 @@ level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial t
 level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
 level=info msg="Collection completed"
 level=info msg="Done, starting HTTP server"
-level=info msg="Starting HTTP server" address=127.0.0.1:8104
-level=warn msg="Error while configuring HTTP transport for history request" error="failed to create HTTP transport for 'http://127.0.0.1:9104': open /xxx/yyy/ca.pem: no such file or directory" uri=http://127.0.0.1:9104 worker=1
+level=info msg="Starting HTTP server" address=127.0.0.1:8111
+level=warn msg="Error while configuring HTTP transport for history request" error="failed to create HTTP transport for 'http://127.0.0.1:9111': open /xxx/yyy/ca.pem: no such file or directory" uri=http://127.0.0.1:9111 worker=1
 level=info msg="Shutting down HTTP server"
 level=info msg="HTTP server shut down"
 level=info msg="Removing PID file" path=karma.pid
@@ -33,7 +33,7 @@ alertmanager:
       uri: http://127.0.0.1
 listen:
   address: 127.0.0.1
-  port: 8104
+  port: 8111
 history:
   enabled: true
   workers: 1
@@ -47,8 +47,8 @@ history:
 -- query.json --
 {
     "sources": [
-        "http://127.0.0.1:9104",
-        "http://127.0.0.1:9104"
+        "http://127.0.0.1:9111",
+        "http://127.0.0.1:9111"
     ],
     "labels": {
         "alertname": "Fake Alert"
@@ -64,7 +64,7 @@ I=0
 while [ ! -f karma.pid ] && [ $I -lt 30 ]; do sleep 1; I=$((I+1)); done
 
 sleep 5
-curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8104/history.json
+curl -s -f -o /dev/null -XPOST -d @query.json http://127.0.0.1:8111/history.json
 cat karma.pid | xargs kill
 cat prometheus.pid | xargs kill
 
@@ -118,13 +118,13 @@ func main() {
 	http.HandleFunc("/api/v1/labels", labelNames)
 	http.HandleFunc("/api/v1/query_range", query)
 
-	listener, err := net.Listen("tcp", "127.0.0.1:9104")
+	listener, err := net.Listen("tcp", "127.0.0.1:9111")
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	server := &http.Server{
-		Addr: "127.0.0.1:9104",
+		Addr: "127.0.0.1:9111",
 	}
 
 	go func() {