diff --git a/cmd/karma/main.go b/cmd/karma/main.go
index cb5708b91..3e127274b 100644
--- a/cmd/karma/main.go
+++ b/cmd/karma/main.go
@@ -99,6 +99,13 @@ func setupRouter(router *chi.Mux, historyPoller *historyPoller) {
 		MaxAge:           300,
 	}))
 
+	router.NotFound(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+		contentText(w)
+		w.WriteHeader(http.StatusNotFound)
+		_, _ = w.Write([]byte("Not found"))
+	})
+
 	allowAuthBypass := []string{
 		getViewURL("/health"),
 		getViewURL("/metrics"),
diff --git a/cmd/karma/tests/testscript/108_alert_history_proxy_url.txt b/cmd/karma/tests/testscript/108_alert_history_proxy_url.txt
index 0e1fd1291..7e46cf4c3 100644
--- a/cmd/karma/tests/testscript/108_alert_history_proxy_url.txt
+++ b/cmd/karma/tests/testscript/108_alert_history_proxy_url.txt
@@ -28,8 +28,8 @@ level=info msg="Removing PID file" path=karma.pid
 -- query.json --
 {
     "sources": [
-        "http://127.0.0.1:9099",
-        "http://127.0.0.1:9099"
+        "http://127.0.0.1:6108",
+        "http://127.0.0.1:6108"
     ],
     "labels": {
         "alertname": "Fake Alert"