feat(backend): add /robots.txt endpoint

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [unreleased]
+
+### Added
+
+- Add `/robots.txt` to block search engine crawlers.
+
 ## v0.79
 
 ### Fixed

cmd/karma/main.go

@@ -128,6 +128,7 @@ func setupRouter(router *chi.Mux) {
 
 	router.Get(getViewURL("/"), index)
 	router.Get(getViewURL("/health"), pong)
+	router.Get(getViewURL("/robots.txt"), robots)
 	router.Get(getViewURL("/metrics"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		h := promhttp.Handler()
 		h.ServeHTTP(w, r)

cmd/karma/views.go

@@ -41,6 +41,10 @@ func pong(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte("Pong\n"))
 }
 
+func robots(w http.ResponseWriter, r *http.Request) {
+	_, _ = w.Write([]byte("User-agent: *\nDisallow: /\n"))
+}
+
 func compressResponse(data []byte) ([]byte, error) {
 	var b bytes.Buffer
 	// this only fails if we pass unsupported level (3 is valid)

cmd/karma/views_test.go

@@ -75,6 +75,18 @@ func TestHealth(t *testing.T) {
 	}
 }
 
+func TestRobots(t *testing.T) {
+	mockConfig()
+	r := testRouter()
+	setupRouter(r)
+	req := httptest.NewRequest("GET", "/robots.txt", nil)
+	resp := httptest.NewRecorder()
+	r.ServeHTTP(resp, req)
+	if resp.Code != http.StatusOK {
+		t.Errorf("GET /robots.txt returned status %d", resp.Code)
+	}
+}
+
 func TestHealthPrefix(t *testing.T) {
 	os.Setenv("LISTEN_PREFIX", "/prefix")
 	defer os.Unsetenv("LISTEN_PREFIX")