diff --git a/CHANGELOG.md b/CHANGELOG.md index 3602eee5b..ce6252c59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [unreleased] + +### Added + +- Add `/robots.txt` to block search engine crawlers. + ## v0.79 ### Fixed diff --git a/cmd/karma/main.go b/cmd/karma/main.go index 2ead680e9..f28090ce5 100644 --- a/cmd/karma/main.go +++ b/cmd/karma/main.go @@ -128,6 +128,7 @@ func setupRouter(router *chi.Mux) { router.Get(getViewURL("/"), index) router.Get(getViewURL("/health"), pong) + router.Get(getViewURL("/robots.txt"), robots) router.Get(getViewURL("/metrics"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { h := promhttp.Handler() h.ServeHTTP(w, r) diff --git a/cmd/karma/views.go b/cmd/karma/views.go index 28415b9c1..05033464a 100644 --- a/cmd/karma/views.go +++ b/cmd/karma/views.go @@ -41,6 +41,10 @@ func pong(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("Pong\n")) } +func robots(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("User-agent: *\nDisallow: /\n")) +} + func compressResponse(data []byte) ([]byte, error) { var b bytes.Buffer // this only fails if we pass unsupported level (3 is valid) diff --git a/cmd/karma/views_test.go b/cmd/karma/views_test.go index 55d9699c8..a69212baa 100644 --- a/cmd/karma/views_test.go +++ b/cmd/karma/views_test.go @@ -75,6 +75,18 @@ func TestHealth(t *testing.T) { } } +func TestRobots(t *testing.T) { + mockConfig() + r := testRouter() + setupRouter(r) + req := httptest.NewRequest("GET", "/robots.txt", nil) + resp := httptest.NewRecorder() + r.ServeHTTP(resp, req) + if resp.Code != http.StatusOK { + t.Errorf("GET /robots.txt returned status %d", resp.Code) + } +} + func TestHealthPrefix(t *testing.T) { os.Setenv("LISTEN_PREFIX", "/prefix") defer os.Unsetenv("LISTEN_PREFIX")