diff --git a/README.md b/README.md index 3b2507c..16bb196 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ Web API: * `GET /readyz` used by Kubernetes readiness probe * `POST /readyz/enable` signals the Kubernetes LB that this instance is ready to receive traffic * `POST /readyz/disable` signals the Kubernetes LB to stop sending requests to this instance +* `POST /fault_injection/enable` makes this instance respond with HTTP 500 to all application endpoints (probes, metrics, pprof and the `/fault_injection/*` control endpoints stay healthy) — useful for testing client-side circuit breakers / outlier detection against a single "sick" replica +* `POST /fault_injection/disable` restores normal responses +* `GET /fault_injection/status` returns the current fault injection state (`enabled` or `disabled`) * `GET /status/{code}` returns the status code * `GET /panic` crashes the process with exit code 255 * `POST /echo` forwards the call to the backend service and echos the posted content diff --git a/pkg/api/http/docs/docs.go b/pkg/api/http/docs/docs.go index a576e74..7a56b79 100644 --- a/pkg/api/http/docs/docs.go +++ b/pkg/api/http/docs/docs.go @@ -19,6 +19,7 @@ const docTemplate = `{ }, "version": "{{.Version}}" }, + "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { "/": { @@ -263,6 +264,77 @@ const docTemplate = `{ } } }, + "/fault_injection/disable": { + "post": { + "description": "restores normal responses", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Fault Injection" + ], + "summary": "Disable fault injection", + "responses": { + "202": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, + "/fault_injection/enable": { + "post": { + "description": "makes the server respond with HTTP 500 for all non-probe endpoints", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Fault Injection" + ], + "summary": "Enable fault injection", + "responses": { + "202": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, + "/fault_injection/status": { + "get": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Fault Injection" + ], + "summary": "Get fault injection status", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/headers": { "get": { "description": "returns a JSON array with the request HTTP headers", diff --git a/pkg/api/http/docs/swagger.json b/pkg/api/http/docs/swagger.json index 0e0ea5b..dae9825 100644 --- a/pkg/api/http/docs/swagger.json +++ b/pkg/api/http/docs/swagger.json @@ -261,6 +261,77 @@ } } }, + "/fault_injection/disable": { + "post": { + "description": "restores normal responses", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Fault Injection" + ], + "summary": "Disable fault injection", + "responses": { + "202": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, + "/fault_injection/enable": { + "post": { + "description": "makes the server respond with HTTP 500 for all non-probe endpoints", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Fault Injection" + ], + "summary": "Enable fault injection", + "responses": { + "202": { + "description": "OK", + "schema": { + "type": "string" + } + } + } + } + }, + "/fault_injection/status": { + "get": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Fault Injection" + ], + "summary": "Get fault injection status", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/headers": { "get": { "description": "returns a JSON array with the request HTTP headers", diff --git a/pkg/api/http/docs/swagger.yaml b/pkg/api/http/docs/swagger.yaml index 91f9d17..11233da 100644 --- a/pkg/api/http/docs/swagger.yaml +++ b/pkg/api/http/docs/swagger.yaml @@ -214,6 +214,52 @@ paths: summary: Environment tags: - HTTP API + /fault_injection/disable: + post: + consumes: + - application/json + description: restores normal responses + produces: + - application/json + responses: + "202": + description: OK + schema: + type: string + summary: Disable fault injection + tags: + - Fault Injection + /fault_injection/enable: + post: + consumes: + - application/json + description: makes the server respond with HTTP 500 for all non-probe endpoints + produces: + - application/json + responses: + "202": + description: OK + schema: + type: string + summary: Enable fault injection + tags: + - Fault Injection + /fault_injection/status: + get: + consumes: + - application/json + produces: + - application/json + responses: + "200": + description: OK + schema: + additionalProperties: + type: string + type: object + summary: Get fault injection status + tags: + - Fault Injection /headers: get: consumes: diff --git a/pkg/api/http/fault_injection.go b/pkg/api/http/fault_injection.go new file mode 100644 index 0000000..c001d36 --- /dev/null +++ b/pkg/api/http/fault_injection.go @@ -0,0 +1,104 @@ +package http + +import ( + "net/http" + "strings" + "sync/atomic" +) + +// faultInjection toggles a process-wide fault-injection mode in which +// the server responds with HTTP 500 for application endpoints. It is +// intended for testing client-side circuit breakers / outlier detection +// against a single "sick" replica. +var faultInjection int32 + +// faultInjectionExcluded returns true if the given request path should +// bypass fault injection. Kubernetes probes, metrics, pprof and the +// control endpoints themselves stay functional so the pod is not torn +// down by the platform while a circuit breaker detects the fault. +func faultInjectionExcluded(p string) bool { + switch { + case strings.HasPrefix(p, "/fault_injection"): + return true + case p == "/healthz", p == "/readyz": + return true + case p == "/metrics": + return true + case strings.HasPrefix(p, "/debug/"): + return true + } + return false +} + +func faultInjectionMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(r.URL.Path) { + http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError) + return + } + next.ServeHTTP(w, r) + }) +} + +// faultInjectionMiddleware is a method form that strips the configured +// path prefix before consulting the exclusion list, so the same set of +// excluded routes works regardless of whether --prefix is set. +func (s *Server) faultInjectionMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p := r.URL.Path + if prefix := s.config.Prefix; prefix != "" && prefix != "/" { + if trimmed := strings.TrimPrefix(p, prefix); trimmed != p { + if trimmed == "" { + trimmed = "/" + } + p = trimmed + } + } + if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(p) { + http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError) + return + } + next.ServeHTTP(w, r) + }) +} + +// EnableFaultInjection godoc +// @Summary Enable fault injection +// @Description makes the server respond with HTTP 500 for all non-probe endpoints +// @Tags Fault Injection +// @Accept json +// @Produce json +// @Router /fault_injection/enable [post] +// @Success 202 {string} string "OK" +func (s *Server) enableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) { + atomic.StoreInt32(&faultInjection, 1) + s.JSONResponseCode(w, r, map[string]string{"fault_injection": "enabled"}, http.StatusAccepted) +} + +// DisableFaultInjection godoc +// @Summary Disable fault injection +// @Description restores normal responses +// @Tags Fault Injection +// @Accept json +// @Produce json +// @Router /fault_injection/disable [post] +// @Success 202 {string} string "OK" +func (s *Server) disableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) { + atomic.StoreInt32(&faultInjection, 0) + s.JSONResponseCode(w, r, map[string]string{"fault_injection": "disabled"}, http.StatusAccepted) +} + +// FaultInjectionStatus godoc +// @Summary Get fault injection status +// @Tags Fault Injection +// @Accept json +// @Produce json +// @Router /fault_injection/status [get] +// @Success 200 {object} map[string]string +func (s *Server) faultInjectionStatusHandler(w http.ResponseWriter, r *http.Request) { + state := "disabled" + if atomic.LoadInt32(&faultInjection) == 1 { + state = "enabled" + } + s.JSONResponse(w, r, map[string]string{"fault_injection": state}) +} diff --git a/pkg/api/http/fault_injection_test.go b/pkg/api/http/fault_injection_test.go new file mode 100644 index 0000000..10f3cea --- /dev/null +++ b/pkg/api/http/fault_injection_test.go @@ -0,0 +1,130 @@ +package http + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestFaultInjection_EnableDisable(t *testing.T) { + defer atomic.StoreInt32(&faultInjection, 0) + srv := NewMockServer() + + rr := httptest.NewRecorder() + req, _ := http.NewRequest("POST", "/fault_injection/enable", nil) + http.HandlerFunc(srv.enableFaultInjectionHandler).ServeHTTP(rr, req) + if rr.Code != http.StatusAccepted { + t.Fatalf("enable: got %d want %d", rr.Code, http.StatusAccepted) + } + if ct := rr.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Errorf("enable: Content-Type = %q, want application/json", ct) + } + if !strings.Contains(rr.Body.String(), `"enabled"`) { + t.Errorf("enable: expected enabled in body, got: %s", rr.Body.String()) + } + if atomic.LoadInt32(&faultInjection) != 1 { + t.Fatalf("faultInjection flag not set after enable") + } + + rr = httptest.NewRecorder() + req, _ = http.NewRequest("POST", "/fault_injection/disable", nil) + http.HandlerFunc(srv.disableFaultInjectionHandler).ServeHTTP(rr, req) + if rr.Code != http.StatusAccepted { + t.Fatalf("disable: got %d want %d", rr.Code, http.StatusAccepted) + } + if ct := rr.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Errorf("disable: Content-Type = %q, want application/json", ct) + } + if !strings.Contains(rr.Body.String(), `"disabled"`) { + t.Errorf("disable: expected disabled in body, got: %s", rr.Body.String()) + } + if atomic.LoadInt32(&faultInjection) != 0 { + t.Fatalf("faultInjection flag not cleared after disable") + } +} + +func TestFaultInjection_StatusHandler(t *testing.T) { + defer atomic.StoreInt32(&faultInjection, 0) + srv := NewMockServer() + + rr := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/fault_injection/status", nil) + http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status: got %d want %d", rr.Code, http.StatusOK) + } + if !strings.Contains(rr.Body.String(), "disabled") { + t.Errorf("expected disabled in body, got: %s", rr.Body.String()) + } + + atomic.StoreInt32(&faultInjection, 1) + rr = httptest.NewRecorder() + http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req) + if !strings.Contains(rr.Body.String(), "enabled") { + t.Errorf("expected enabled in body, got: %s", rr.Body.String()) + } +} + +func TestFaultInjectionMiddleware(t *testing.T) { + ok := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) + }) + h := faultInjectionMiddleware(ok) + + cases := []struct { + name string + path string + injected bool + wantStatus int + }{ + {"disabled passes through", "/", false, http.StatusOK}, + {"enabled returns 500 for app path", "/", true, http.StatusInternalServerError}, + {"enabled returns 500 for arbitrary path", "/api/info", true, http.StatusInternalServerError}, + {"enabled excludes healthz", "/healthz", true, http.StatusOK}, + {"enabled excludes readyz", "/readyz", true, http.StatusOK}, + {"enabled excludes metrics", "/metrics", true, http.StatusOK}, + {"enabled excludes debug pprof", "/debug/pprof/", true, http.StatusOK}, + {"enabled excludes fault_injection control", "/fault_injection/disable", true, http.StatusOK}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.injected { + atomic.StoreInt32(&faultInjection, 1) + } else { + atomic.StoreInt32(&faultInjection, 0) + } + defer atomic.StoreInt32(&faultInjection, 0) + + req, _ := http.NewRequest("GET", tc.path, nil) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + if rr.Code != tc.wantStatus { + t.Fatalf("path %s: got %d want %d", tc.path, rr.Code, tc.wantStatus) + } + }) + } +} + +func TestFaultInjectionExcluded(t *testing.T) { + cases := map[string]bool{ + "/": false, + "/api/info": false, + "/healthz": true, + "/readyz": true, + "/metrics": true, + "/debug/pprof/": true, + "/fault_injection/enable": true, + "/fault_injection/disable": true, + "/fault_injection/status": true, + "/healthzz": false, + } + for p, want := range cases { + if got := faultInjectionExcluded(p); got != want { + t.Errorf("faultInjectionExcluded(%q) = %v, want %v", p, got, want) + } + } +} diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index c0c4948..27630ae 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -153,6 +153,9 @@ func (s *Server) registerHandlers() { s.router.HandleFunc(s.prefixedPath("/readyz"), s.readyzHandler).Methods("GET") s.router.HandleFunc(s.prefixedPath("/readyz/enable"), s.enableReadyHandler).Methods("POST") s.router.HandleFunc(s.prefixedPath("/readyz/disable"), s.disableReadyHandler).Methods("POST") + s.router.HandleFunc(s.prefixedPath("/fault_injection/enable"), s.enableFaultInjectionHandler).Methods("POST") + s.router.HandleFunc(s.prefixedPath("/fault_injection/disable"), s.disableFaultInjectionHandler).Methods("POST") + s.router.HandleFunc(s.prefixedPath("/fault_injection/status"), s.faultInjectionStatusHandler).Methods("GET") s.router.HandleFunc(s.prefixedPath("/panic"), s.panicHandler).Methods("GET") s.router.HandleFunc(s.prefixedPath("/status/{code:[0-9]+}"), s.statusHandler).Methods("GET", "POST", "PUT").Name("status") s.router.HandleFunc(s.prefixedPath("/store"), s.storeWriteHandler).Methods("POST", "PUT") @@ -189,6 +192,7 @@ func (s *Server) registerMiddlewares() { httpLogger := NewLoggingMiddleware(s.logger) s.router.Use(httpLogger.Handler) s.router.Use(versionMiddleware) + s.router.Use(s.faultInjectionMiddleware) if s.config.RandomDelay { randomDelayer := NewRandomDelayMiddleware(s.config.RandomDelayMin, s.config.RandomDelayMax, s.config.RandomDelayUnit) s.router.Use(randomDelayer.Handler)