diff --git a/pkg/api/http/fault_injection.go b/pkg/api/http/fault_injection.go new file mode 100644 index 0000000..89113e6 --- /dev/null +++ b/pkg/api/http/fault_injection.go @@ -0,0 +1,106 @@ +package http + +import ( + "net/http" + "strings" + "sync/atomic" +) + +// faultInjection toggles a process-wide fault-injection mode in which +// the server responds with HTTP 500 for application endpoints. It is +// intended for testing client-side circuit breakers / outlier detection +// against a single "sick" replica. +var faultInjection int32 + +// faultInjectionExcluded returns true if the given request path should +// bypass fault injection. Kubernetes probes, metrics, pprof and the +// control endpoints themselves stay functional so the pod is not torn +// down by the platform while a circuit breaker detects the fault. +func faultInjectionExcluded(p string) bool { + switch { + case strings.HasPrefix(p, "/fault_injection"): + return true + case p == "/healthz", p == "/readyz": + return true + case p == "/metrics": + return true + case strings.HasPrefix(p, "/debug/"): + return true + } + return false +} + +func faultInjectionMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(r.URL.Path) { + http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError) + return + } + next.ServeHTTP(w, r) + }) +} + +// faultInjectionMiddleware is a method form that strips the configured +// path prefix before consulting the exclusion list, so the same set of +// excluded routes works regardless of whether --prefix is set. +func (s *Server) faultInjectionMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p := r.URL.Path + if prefix := s.config.Prefix; prefix != "" && prefix != "/" { + if trimmed := strings.TrimPrefix(p, prefix); trimmed != p { + if trimmed == "" { + trimmed = "/" + } + p = trimmed + } + } + if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(p) { + http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError) + return + } + next.ServeHTTP(w, r) + }) +} + +// EnableFaultInjection godoc +// @Summary Enable fault injection +// @Description makes the server respond with HTTP 500 for all non-probe endpoints +// @Tags Fault Injection +// @Accept json +// @Produce json +// @Router /fault_injection/enable [post] +// @Success 202 {string} string "OK" +func (s *Server) enableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) { + atomic.StoreInt32(&faultInjection, 1) + w.WriteHeader(http.StatusAccepted) + s.JSONResponse(w, r, map[string]string{"fault_injection": "enabled"}) +} + +// DisableFaultInjection godoc +// @Summary Disable fault injection +// @Description restores normal responses +// @Tags Fault Injection +// @Accept json +// @Produce json +// @Router /fault_injection/disable [post] +// @Success 202 {string} string "OK" +func (s *Server) disableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) { + atomic.StoreInt32(&faultInjection, 0) + w.WriteHeader(http.StatusAccepted) + s.JSONResponse(w, r, map[string]string{"fault_injection": "disabled"}) +} + +// FaultInjectionStatus godoc +// @Summary Get fault injection status +// @Tags Fault Injection +// @Accept json +// @Produce json +// @Router /fault_injection/status [get] +// @Success 200 {object} map[string]string +func (s *Server) faultInjectionStatusHandler(w http.ResponseWriter, r *http.Request) { + state := "disabled" + if atomic.LoadInt32(&faultInjection) == 1 { + state = "enabled" + } + s.JSONResponse(w, r, map[string]string{"fault_injection": state}) +} diff --git a/pkg/api/http/fault_injection_test.go b/pkg/api/http/fault_injection_test.go new file mode 100644 index 0000000..80bfbea --- /dev/null +++ b/pkg/api/http/fault_injection_test.go @@ -0,0 +1,118 @@ +package http + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestFaultInjection_EnableDisable(t *testing.T) { + defer atomic.StoreInt32(&faultInjection, 0) + srv := NewMockServer() + + rr := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/fault_injection/enable", nil) + http.HandlerFunc(srv.enableFaultInjectionHandler).ServeHTTP(rr, req) + if rr.Code != http.StatusAccepted { + t.Fatalf("enable: got %d want %d", rr.Code, http.StatusAccepted) + } + if atomic.LoadInt32(&faultInjection) != 1 { + t.Fatalf("faultInjection flag not set after enable") + } + + rr = httptest.NewRecorder() + req, _ = http.NewRequest("POST", "/fault_injection/disable", nil) + http.HandlerFunc(srv.disableFaultInjectionHandler).ServeHTTP(rr, req) + if rr.Code != http.StatusAccepted { + t.Fatalf("disable: got %d want %d", rr.Code, http.StatusAccepted) + } + if atomic.LoadInt32(&faultInjection) != 0 { + t.Fatalf("faultInjection flag not cleared after disable") + } +} + +func TestFaultInjection_StatusHandler(t *testing.T) { + defer atomic.StoreInt32(&faultInjection, 0) + srv := NewMockServer() + + rr := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/fault_injection/status", nil) + http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status: got %d want %d", rr.Code, http.StatusOK) + } + if !strings.Contains(rr.Body.String(), "disabled") { + t.Errorf("expected disabled in body, got: %s", rr.Body.String()) + } + + atomic.StoreInt32(&faultInjection, 1) + rr = httptest.NewRecorder() + http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req) + if !strings.Contains(rr.Body.String(), "enabled") { + t.Errorf("expected enabled in body, got: %s", rr.Body.String()) + } +} + +func TestFaultInjectionMiddleware(t *testing.T) { + ok := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) + }) + h := faultInjectionMiddleware(ok) + + cases := []struct { + name string + path string + injected bool + wantStatus int + }{ + {"disabled passes through", "/", false, http.StatusOK}, + {"enabled returns 500 for app path", "/", true, http.StatusInternalServerError}, + {"enabled returns 500 for arbitrary path", "/api/info", true, http.StatusInternalServerError}, + {"enabled excludes healthz", "/healthz", true, http.StatusOK}, + {"enabled excludes readyz", "/readyz", true, http.StatusOK}, + {"enabled excludes metrics", "/metrics", true, http.StatusOK}, + {"enabled excludes debug pprof", "/debug/pprof/", true, http.StatusOK}, + {"enabled excludes fault_injection control", "/fault_injection/disable", true, http.StatusOK}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.injected { + atomic.StoreInt32(&faultInjection, 1) + } else { + atomic.StoreInt32(&faultInjection, 0) + } + defer atomic.StoreInt32(&faultInjection, 0) + + req, _ := http.NewRequest("GET", tc.path, nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + if rr.Code != tc.wantStatus { + t.Fatalf("path %s: got %d want %d", tc.path, rr.Code, tc.wantStatus) + } + }) + } +} + +func TestFaultInjectionExcluded(t *testing.T) { + cases := map[string]bool{ + "/": false, + "/api/info": false, + "/healthz": true, + "/readyz": true, + "/metrics": true, + "/debug/pprof/": true, + "/fault_injection/enable": true, + "/fault_injection/disable": true, + "/fault_injection/status": true, + "/healthzz": false, + } + for p, want := range cases { + if got := faultInjectionExcluded(p); got != want { + t.Errorf("faultInjectionExcluded(%q) = %v, want %v", p, got, want) + } + } +} diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index c0c4948..27630ae 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -153,6 +153,9 @@ func (s *Server) registerHandlers() { s.router.HandleFunc(s.prefixedPath("/readyz"), s.readyzHandler).Methods("GET") s.router.HandleFunc(s.prefixedPath("/readyz/enable"), s.enableReadyHandler).Methods("POST") s.router.HandleFunc(s.prefixedPath("/readyz/disable"), s.disableReadyHandler).Methods("POST") + s.router.HandleFunc(s.prefixedPath("/fault_injection/enable"), s.enableFaultInjectionHandler).Methods("POST") + s.router.HandleFunc(s.prefixedPath("/fault_injection/disable"), s.disableFaultInjectionHandler).Methods("POST") + s.router.HandleFunc(s.prefixedPath("/fault_injection/status"), s.faultInjectionStatusHandler).Methods("GET") s.router.HandleFunc(s.prefixedPath("/panic"), s.panicHandler).Methods("GET") s.router.HandleFunc(s.prefixedPath("/status/{code:[0-9]+}"), s.statusHandler).Methods("GET", "POST", "PUT").Name("status") s.router.HandleFunc(s.prefixedPath("/store"), s.storeWriteHandler).Methods("POST", "PUT") @@ -189,6 +192,7 @@ func (s *Server) registerMiddlewares() { httpLogger := NewLoggingMiddleware(s.logger) s.router.Use(httpLogger.Handler) s.router.Use(versionMiddleware) + s.router.Use(s.faultInjectionMiddleware) if s.config.RandomDelay { randomDelayer := NewRandomDelayMiddleware(s.config.RandomDelayMin, s.config.RandomDelayMax, s.config.RandomDelayUnit) s.router.Use(randomDelayer.Handler)