github/podinfo
1
0
Fork 0
mirror of https://github.com/stefanprodan/podinfo.git synced 2026-08-19 04:06:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #492 from dkulchinsky/feat/fault-injection

Browse Source 
feat(http): add fault injection mode for circuit breaker testing
This commit is contained in:
Stefan Prodan
2026-06-04 00:48:24 +03:00
committed by GitHub
parent 8a12123ba3 04fa4f98ac
commit 6a83cf4376
7 changed files with 430 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3
View File
@@ -37,6 +37,9 @@ Web API:
* `GET /readyz` used by Kubernetes readiness probe
* `POST /readyz/enable` signals the Kubernetes LB that this instance is ready to receive traffic
* `POST /readyz/disable` signals the Kubernetes LB to stop sending requests to this instance
* `POST /fault_injection/enable` makes this instance respond with HTTP 500 to all application endpoints (probes, metrics, pprof and the `/fault_injection/*` control endpoints stay healthy) — useful for testing client-side circuit breakers / outlier detection against a single "sick" replica
* `POST /fault_injection/disable` restores normal responses
* `GET /fault_injection/status` returns the current fault injection state (`enabled` or `disabled`)
* `GET /status/{code}` returns the status code
* `GET /panic` crashes the process with exit code 255
* `POST /echo` forwards the call to the backend service and echos the posted content
pkg/api/http/docs/docs.go
+72
View File
@@ -19,6 +19,7 @@ const docTemplate = `{
},
"version": "{{.Version}}"
},
"host": "{{.Host}}",
"basePath": "{{.BasePath}}",
"paths": {
"/": {
@@ -263,6 +264,77 @@ const docTemplate = `{
}
}
},
"/fault_injection/disable": {
"post": {
"description": "restores normal responses",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"Fault Injection"
],
"summary": "Disable fault injection",
"responses": {
"202": {
"description": "OK",
"schema": {
"type": "string"
}
}
}
}
},
"/fault_injection/enable": {
"post": {
"description": "makes the server respond with HTTP 500 for all non-probe endpoints",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"Fault Injection"
],
"summary": "Enable fault injection",
"responses": {
"202": {
"description": "OK",
"schema": {
"type": "string"
}
}
}
}
},
"/fault_injection/status": {
"get": {
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"Fault Injection"
],
"summary": "Get fault injection status",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/headers": {
"get": {
"description": "returns a JSON array with the request HTTP headers",
pkg/api/http/docs/swagger.json
+71
View File
@@ -261,6 +261,77 @@
}
}
},
"/fault_injection/disable": {
"post": {
"description": "restores normal responses",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"Fault Injection"
],
"summary": "Disable fault injection",
"responses": {
"202": {
"description": "OK",
"schema": {
"type": "string"
}
}
}
}
},
"/fault_injection/enable": {
"post": {
"description": "makes the server respond with HTTP 500 for all non-probe endpoints",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"Fault Injection"
],
"summary": "Enable fault injection",
"responses": {
"202": {
"description": "OK",
"schema": {
"type": "string"
}
}
}
}
},
"/fault_injection/status": {
"get": {
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"Fault Injection"
],
"summary": "Get fault injection status",
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/headers": {
"get": {
"description": "returns a JSON array with the request HTTP headers",
pkg/api/http/docs/swagger.yaml
+46
View File
@@ -214,6 +214,52 @@ paths:
summary: Environment
tags:
- HTTP API
/fault_injection/disable:
post:
consumes:
- application/json
description: restores normal responses
produces:
- application/json
responses:
"202":
description: OK
schema:
type: string
summary: Disable fault injection
tags:
- Fault Injection
/fault_injection/enable:
post:
consumes:
- application/json
description: makes the server respond with HTTP 500 for all non-probe endpoints
produces:
- application/json
responses:
"202":
description: OK
schema:
type: string
summary: Enable fault injection
tags:
- Fault Injection
/fault_injection/status:
get:
consumes:
- application/json
produces:
- application/json
responses:
"200":
description: OK
schema:
additionalProperties:
type: string
type: object
summary: Get fault injection status
tags:
- Fault Injection
/headers:
get:
consumes:
pkg/api/http/fault_injection.go
+104
View File
@@ -0,0 +1,104 @@
package http
import (
"net/http"
"strings"
"sync/atomic"
)
// faultInjection toggles a process-wide fault-injection mode in which
// the server responds with HTTP 500 for application endpoints. It is
// intended for testing client-side circuit breakers / outlier detection
// against a single "sick" replica.
var faultInjection int32
// faultInjectionExcluded returns true if the given request path should
// bypass fault injection. Kubernetes probes, metrics, pprof and the
// control endpoints themselves stay functional so the pod is not torn
// down by the platform while a circuit breaker detects the fault.
func faultInjectionExcluded(p string) bool {
switch {
case strings.HasPrefix(p, "/fault_injection"):
return true
case p == "/healthz", p == "/readyz":
return true
case p == "/metrics":
return true
case strings.HasPrefix(p, "/debug/"):
return true
}
return false
}
func faultInjectionMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(r.URL.Path) {
http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError)
return
}
next.ServeHTTP(w, r)
})
}
// faultInjectionMiddleware is a method form that strips the configured
// path prefix before consulting the exclusion list, so the same set of
// excluded routes works regardless of whether --prefix is set.
func (s *Server) faultInjectionMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
p := r.URL.Path
if prefix := s.config.Prefix; prefix != "" && prefix != "/" {
if trimmed := strings.TrimPrefix(p, prefix); trimmed != p {
if trimmed == "" {
trimmed = "/"
}
p = trimmed
}
}
if atomic.LoadInt32(&faultInjection) == 1 && !faultInjectionExcluded(p) {
http.Error(w, `{"status":"fault injection enabled"}`, http.StatusInternalServerError)
return
}
next.ServeHTTP(w, r)
})
}
// EnableFaultInjection godoc
// @Summary Enable fault injection
// @Description makes the server respond with HTTP 500 for all non-probe endpoints
// @Tags Fault Injection
// @Accept json
// @Produce json
// @Router /fault_injection/enable [post]
// @Success 202 {string} string "OK"
func (s *Server) enableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) {
atomic.StoreInt32(&faultInjection, 1)
s.JSONResponseCode(w, r, map[string]string{"fault_injection": "enabled"}, http.StatusAccepted)
}
// DisableFaultInjection godoc
// @Summary Disable fault injection
// @Description restores normal responses
// @Tags Fault Injection
// @Accept json
// @Produce json
// @Router /fault_injection/disable [post]
// @Success 202 {string} string "OK"
func (s *Server) disableFaultInjectionHandler(w http.ResponseWriter, r *http.Request) {
atomic.StoreInt32(&faultInjection, 0)
s.JSONResponseCode(w, r, map[string]string{"fault_injection": "disabled"}, http.StatusAccepted)
}
// FaultInjectionStatus godoc
// @Summary Get fault injection status
// @Tags Fault Injection
// @Accept json
// @Produce json
// @Router /fault_injection/status [get]
// @Success 200 {object} map[string]string
func (s *Server) faultInjectionStatusHandler(w http.ResponseWriter, r *http.Request) {
state := "disabled"
if atomic.LoadInt32(&faultInjection) == 1 {
state = "enabled"
}
s.JSONResponse(w, r, map[string]string{"fault_injection": state})
}
pkg/api/http/fault_injection_test.go
+130
View File
@@ -0,0 +1,130 @@
package http
import (
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
)
func TestFaultInjection_EnableDisable(t *testing.T) {
defer atomic.StoreInt32(&faultInjection, 0)
srv := NewMockServer()
rr := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "/fault_injection/enable", nil)
http.HandlerFunc(srv.enableFaultInjectionHandler).ServeHTTP(rr, req)
if rr.Code != http.StatusAccepted {
t.Fatalf("enable: got %d want %d", rr.Code, http.StatusAccepted)
}
if ct := rr.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
t.Errorf("enable: Content-Type = %q, want application/json", ct)
}
if !strings.Contains(rr.Body.String(), `"enabled"`) {
t.Errorf("enable: expected enabled in body, got: %s", rr.Body.String())
}
if atomic.LoadInt32(&faultInjection) != 1 {
t.Fatalf("faultInjection flag not set after enable")
}
rr = httptest.NewRecorder()
req, _ = http.NewRequest("POST", "/fault_injection/disable", nil)
http.HandlerFunc(srv.disableFaultInjectionHandler).ServeHTTP(rr, req)
if rr.Code != http.StatusAccepted {
t.Fatalf("disable: got %d want %d", rr.Code, http.StatusAccepted)
}
if ct := rr.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
t.Errorf("disable: Content-Type = %q, want application/json", ct)
}
if !strings.Contains(rr.Body.String(), `"disabled"`) {
t.Errorf("disable: expected disabled in body, got: %s", rr.Body.String())
}
if atomic.LoadInt32(&faultInjection) != 0 {
t.Fatalf("faultInjection flag not cleared after disable")
}
}
func TestFaultInjection_StatusHandler(t *testing.T) {
defer atomic.StoreInt32(&faultInjection, 0)
srv := NewMockServer()
rr := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/fault_injection/status", nil)
http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("status: got %d want %d", rr.Code, http.StatusOK)
}
if !strings.Contains(rr.Body.String(), "disabled") {
t.Errorf("expected disabled in body, got: %s", rr.Body.String())
}
atomic.StoreInt32(&faultInjection, 1)
rr = httptest.NewRecorder()
http.HandlerFunc(srv.faultInjectionStatusHandler).ServeHTTP(rr, req)
if !strings.Contains(rr.Body.String(), "enabled") {
t.Errorf("expected enabled in body, got: %s", rr.Body.String())
}
}
func TestFaultInjectionMiddleware(t *testing.T) {
ok := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
})
h := faultInjectionMiddleware(ok)
cases := []struct {
name string
path string
injected bool
wantStatus int
}{
{"disabled passes through", "/", false, http.StatusOK},
{"enabled returns 500 for app path", "/", true, http.StatusInternalServerError},
{"enabled returns 500 for arbitrary path", "/api/info", true, http.StatusInternalServerError},
{"enabled excludes healthz", "/healthz", true, http.StatusOK},
{"enabled excludes readyz", "/readyz", true, http.StatusOK},
{"enabled excludes metrics", "/metrics", true, http.StatusOK},
{"enabled excludes debug pprof", "/debug/pprof/", true, http.StatusOK},
{"enabled excludes fault_injection control", "/fault_injection/disable", true, http.StatusOK},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if tc.injected {
atomic.StoreInt32(&faultInjection, 1)
} else {
atomic.StoreInt32(&faultInjection, 0)
}
defer atomic.StoreInt32(&faultInjection, 0)
req, _ := http.NewRequest("GET", tc.path, nil)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
if rr.Code != tc.wantStatus {
t.Fatalf("path %s: got %d want %d", tc.path, rr.Code, tc.wantStatus)
}
})
}
}
func TestFaultInjectionExcluded(t *testing.T) {
cases := map[string]bool{
"/": false,
"/api/info": false,
"/healthz": true,
"/readyz": true,
"/metrics": true,
"/debug/pprof/": true,
"/fault_injection/enable": true,
"/fault_injection/disable": true,
"/fault_injection/status": true,
"/healthzz": false,
}
for p, want := range cases {
if got := faultInjectionExcluded(p); got != want {
t.Errorf("faultInjectionExcluded(%q) = %v, want %v", p, got, want)
}
}
}
pkg/api/http/server.go
+4
View File
@@ -153,6 +153,9 @@ func (s *Server) registerHandlers() {
s.router.HandleFunc(s.prefixedPath("/readyz"), s.readyzHandler).Methods("GET")
s.router.HandleFunc(s.prefixedPath("/readyz/enable"), s.enableReadyHandler).Methods("POST")
s.router.HandleFunc(s.prefixedPath("/readyz/disable"), s.disableReadyHandler).Methods("POST")
s.router.HandleFunc(s.prefixedPath("/fault_injection/enable"), s.enableFaultInjectionHandler).Methods("POST")
s.router.HandleFunc(s.prefixedPath("/fault_injection/disable"), s.disableFaultInjectionHandler).Methods("POST")
s.router.HandleFunc(s.prefixedPath("/fault_injection/status"), s.faultInjectionStatusHandler).Methods("GET")
s.router.HandleFunc(s.prefixedPath("/panic"), s.panicHandler).Methods("GET")
s.router.HandleFunc(s.prefixedPath("/status/{code:[0-9]+}"), s.statusHandler).Methods("GET", "POST", "PUT").Name("status")
s.router.HandleFunc(s.prefixedPath("/store"), s.storeWriteHandler).Methods("POST", "PUT")
@@ -189,6 +192,7 @@ func (s *Server) registerMiddlewares() {
httpLogger := NewLoggingMiddleware(s.logger)
s.router.Use(httpLogger.Handler)
s.router.Use(versionMiddleware)
s.router.Use(s.faultInjectionMiddleware)
if s.config.RandomDelay {
randomDelayer := NewRandomDelayMiddleware(s.config.RandomDelayMin, s.config.RandomDelayMax, s.config.RandomDelayUnit)
s.router.Use(randomDelayer.Handler)