feat(http): add fault injection mode for circuit breaker testing

Adds a process-wide toggle that makes podinfo respond with HTTP 500 to all application endpoints while keeping Kubernetes probes, metrics, pprof and the control endpoints functional. This allows a single replica to be made selectively 'sick' to test client-side circuit breakers / outlier detection (Envoy, Istio DestinationRule.outlierDetection, etc.) without Kubernetes evicting the pod. New endpoints: POST /fault_injection/enable POST /fault_injection/disable GET /fault_injection/status Includes unit tests for the handlers, the middleware behavior, and the path exclusion list.
2026-08-19 04:06:27 +00:00 · 2026-06-02 23:18:28 -04:00
parent 8a12123ba3
commit 18230d8fc5
3 changed files with 228 additions and 0 deletions
@@ -0,0 +1,106 @@
+package http
+
+import (
+	"net/http"
+	"strings"
+	"sync/atomic"
+)
+
+// faultInjection toggles a process-wide fault-injection mode in which
+// the server responds with HTTP 500 for application endpoints. It is
+// intended for testing client-side circuit breakers / outlier detection
+// against a single "sick" replica.
+var faultInjection int32
+
+// faultInjectionExcluded returns true if the given request path should
+// bypass fault injection. Kubernetes probes, metrics, pprof and the
+// control endpoints themselves stay functional so the pod is not torn
+// down by the platform while a circuit breaker detects the fault.
+func faultInjectionExcluded(p string) bool {
+	switch {
+	case strings.HasPrefix(p, "/fault_injection"):
+		return true
+	case p == "/healthz", p == "/readyz":
+		return true
+	case p == "/metrics":
+		return true
+	case strings.HasPrefix(p, "/debug/"):
+		return true
+	}
+	return false
+}
+
+func faultInjectionMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(r.URL.Path) {
+			http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// faultInjectionMiddleware is a method form that strips the configured
+// path prefix before consulting the exclusion list, so the same set of
+// excluded routes works regardless of whether --prefix is set.
+func (s *Server) faultInjectionMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		p := r.URL.Path
+		if prefix := s.config.Prefix; prefix != "" && prefix != "/" {
+			if trimmed := strings.TrimPrefix(p, prefix); trimmed != p {
+				if trimmed == "" {
+					trimmed = "/"
+				}
+				p = trimmed
+			}
+		}
+		if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(p) {
+			http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// EnableFaultInjection godoc
+// @Summary Enable fault injection
+// @Description makes the server respond with HTTP 500 for all non-probe endpoints
+// @Tags Fault Injection
+// @Accept json
+// @Produce json
+// @Router /fault_injection/enable [post]
+// @Success 202 {string} string "OK"
+func (s *Server) enableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) {
+	atomic.StoreInt32(&faultInjection, 1)
+	w.WriteHeader(http.StatusAccepted)
+	s.JSONResponse(w, r, map[string]string{"fault_injection": "enabled"})
+}
+
+// DisableFaultInjection godoc
+// @Summary Disable fault injection
+// @Description restores normal responses
+// @Tags Fault Injection
+// @Accept json
+// @Produce json
+// @Router /fault_injection/disable [post]
+// @Success 202 {string} string "OK"
+func (s *Server) disableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) {
+	atomic.StoreInt32(&faultInjection, 0)
+	w.WriteHeader(http.StatusAccepted)
+	s.JSONResponse(w, r, map[string]string{"fault_injection": "disabled"})
+}
+
+// FaultInjectionStatus godoc
+// @Summary Get fault injection status
+// @Tags Fault Injection
+// @Accept json
+// @Produce json
+// @Router /fault_injection/status [get]
+// @Success 200 {object} map[string]string
+func (s *Server) faultInjectionStatusHandler(w http.ResponseWriter, r *http.Request) {
+	state := "disabled"
+	if atomic.LoadInt32(&faultInjection) == 1 {
+		state = "enabled"
+	}
+	s.JSONResponse(w, r, map[string]string{"fault_injection": state})
+}
@@ -0,0 +1,118 @@
+package http
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+)
+
+func TestFaultInjection_EnableDisable(t *testing.T) {
+	defer atomic.StoreInt32(&faultInjection, 0)
+	srv := NewMockServer()
+
+	rr := httptest.NewRecorder()
+	req, _ := http.NewRequest("POST", "/fault_injection/enable", nil)
+	http.HandlerFunc(srv.enableFaultInjectionHandler).ServeHTTP(rr, req)
+	if rr.Code != http.StatusAccepted {
+		t.Fatalf("enable: got %d want %d", rr.Code, http.StatusAccepted)
+	}
+	if atomic.LoadInt32(&faultInjection) != 1 {
+		t.Fatalf("faultInjection flag not set after enable")
+	}
+
+	rr = httptest.NewRecorder()
+	req, _ = http.NewRequest("POST", "/fault_injection/disable", nil)
+	http.HandlerFunc(srv.disableFaultInjectionHandler).ServeHTTP(rr, req)
+	if rr.Code != http.StatusAccepted {
+		t.Fatalf("disable: got %d want %d", rr.Code, http.StatusAccepted)
+	}
+	if atomic.LoadInt32(&faultInjection) != 0 {
+		t.Fatalf("faultInjection flag not cleared after disable")
+	}
+}
+
+func TestFaultInjection_StatusHandler(t *testing.T) {
+	defer atomic.StoreInt32(&faultInjection, 0)
+	srv := NewMockServer()
+
+	rr := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "/fault_injection/status", nil)
+	http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status: got %d want %d", rr.Code, http.StatusOK)
+	}
+	if !strings.Contains(rr.Body.String(), "disabled") {
+		t.Errorf("expected disabled in body, got: %s", rr.Body.String())
+	}
+
+	atomic.StoreInt32(&faultInjection, 1)
+	rr = httptest.NewRecorder()
+	http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req)
+	if !strings.Contains(rr.Body.String(), "enabled") {
+		t.Errorf("expected enabled in body, got: %s", rr.Body.String())
+	}
+}
+
+func TestFaultInjectionMiddleware(t *testing.T) {
+	ok := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("ok"))
+	})
+	h := faultInjectionMiddleware(ok)
+
+	cases := []struct {
+		name       string
+		path       string
+		injected   bool
+		wantStatus int
+	}{
+		{"disabled passes through", "/", false, http.StatusOK},
+		{"enabled returns 500 for app path", "/", true, http.StatusInternalServerError},
+		{"enabled returns 500 for arbitrary path", "/api/info", true, http.StatusInternalServerError},
+		{"enabled excludes healthz", "/healthz", true, http.StatusOK},
+		{"enabled excludes readyz", "/readyz", true, http.StatusOK},
+		{"enabled excludes metrics", "/metrics", true, http.StatusOK},
+		{"enabled excludes debug pprof", "/debug/pprof/", true, http.StatusOK},
+		{"enabled excludes fault_injection control", "/fault_injection/disable", true, http.StatusOK},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.injected {
+				atomic.StoreInt32(&faultInjection, 1)
+			} else {
+				atomic.StoreInt32(&faultInjection, 0)
+			}
+			defer atomic.StoreInt32(&faultInjection, 0)
+
+			req, _ := http.NewRequest("GET", tc.path, nil)
+			rr := httptest.NewRecorder()
+			h.ServeHTTP(rr, req)
+			if rr.Code != tc.wantStatus {
+				t.Fatalf("path %s: got %d want %d", tc.path, rr.Code, tc.wantStatus)
+			}
+		})
+	}
+}
+
+func TestFaultInjectionExcluded(t *testing.T) {
+	cases := map[string]bool{
+		"/":                          false,
+		"/api/info":                  false,
+		"/healthz":                   true,
+		"/readyz":                    true,
+		"/metrics":                   true,
+		"/debug/pprof/":              true,
+		"/fault_injection/enable":    true,
+		"/fault_injection/disable":   true,
+		"/fault_injection/status":    true,
+		"/healthzz":                  false,
+	}
+	for p, want := range cases {
+		if got := faultInjectionExcluded(p); got != want {
+			t.Errorf("faultInjectionExcluded(%q) = %v, want %v", p, got, want)
+		}
+	}
+}
@@ -153,6 +153,9 @@ func (s *Server) registerHandlers() {
 	s.router.HandleFunc(s.prefixedPath("/readyz"), s.readyzHandler).Methods("GET")
 	s.router.HandleFunc(s.prefixedPath("/readyz/enable"), s.enableReadyHandler).Methods("POST")
 	s.router.HandleFunc(s.prefixedPath("/readyz/disable"), s.disableReadyHandler).Methods("POST")
+	s.router.HandleFunc(s.prefixedPath("/fault_injection/enable"), s.enableFaultInjectionHandler).Methods("POST")
+	s.router.HandleFunc(s.prefixedPath("/fault_injection/disable"), s.disableFaultInjectionHandler).Methods("POST")
+	s.router.HandleFunc(s.prefixedPath("/fault_injection/status"), s.faultInjectionStatusHandler).Methods("GET")
 	s.router.HandleFunc(s.prefixedPath("/panic"), s.panicHandler).Methods("GET")
 	s.router.HandleFunc(s.prefixedPath("/status/{code:[0-9]+}"), s.statusHandler).Methods("GET", "POST", "PUT").Name("status")
 	s.router.HandleFunc(s.prefixedPath("/store"), s.storeWriteHandler).Methods("POST", "PUT")
@@ -189,6 +192,7 @@ func (s *Server) registerMiddlewares() {
 	httpLogger := NewLoggingMiddleware(s.logger)
 	s.router.Use(httpLogger.Handler)
 	s.router.Use(versionMiddleware)
+	s.router.Use(s.faultInjectionMiddleware)
 	if s.config.RandomDelay {
 		randomDelayer := NewRandomDelayMiddleware(s.config.RandomDelayMin, s.config.RandomDelayMax, s.config.RandomDelayUnit)
 		s.router.Use(randomDelayer.Handler)