Improve structure and re-usability of HTTP interaction recordings

This commit is contained in:
Tobias Gesellchen
2026-02-14 12:39:39 +01:00
parent c8ef1a9de4
commit ef90b4e848
10 changed files with 669 additions and 2 deletions
+15 -2
View File
@@ -30,12 +30,22 @@ func main() {
sm := setup.NewManager(config.serverURL, ds, cm)
server := handlers.NewServer(ds, sm, config.serverURL, config.redact, config.logBody)
recorder := proxy.NewRecorder(config.dataDir)
patternsPath := filepath.Join(config.dataDir, "patterns.json")
patterns, err := proxy.LoadPatterns(patternsPath)
if err == nil && len(patterns) > 0 {
recorder.Patterns = patterns
} else if err != nil {
log.Printf("Warning: Failed to load patterns from %s: %v", patternsPath, err)
}
server.SetRecorder(recorder)
tlsConfig, err := cm.GetServerTLSConfig(config.domains)
if err != nil {
log.Printf("Warning: Failed to setup TLS: %v", err)
}
pyProxy := setupPythonProxy(config.targetURL, config.redact, config.logBody)
pyProxy := setupPythonProxy(config.targetURL, config.redact, config.logBody, recorder)
startDeviceDiscovery(server)
@@ -182,7 +192,7 @@ func initCertificateManager(dataDir string) *certmanager.CertificateManager {
return cm
}
func setupPythonProxy(targetURL string, redact, logBody bool) *httputil.ReverseProxy {
func setupPythonProxy(targetURL string, redact, logBody bool, recorder *proxy.Recorder) *httputil.ReverseProxy {
target, err := url.Parse(targetURL)
if err != nil {
log.Fatalf("Failed to parse target URL: %v", err)
@@ -197,6 +207,7 @@ func setupPythonProxy(targetURL string, redact, logBody bool) *httputil.ReverseP
currentLp := proxy.NewLoggingProxy(target.String(), redact)
currentLp.LogBody = logBody
currentLp.SetRecorder(recorder)
currentLp.LogResponse(res)
return nil
@@ -208,6 +219,7 @@ func setupPythonProxy(targetURL string, redact, logBody bool) *httputil.ReverseP
currentLp := proxy.NewLoggingProxy(target.String(), redact)
currentLp.LogBody = logBody
currentLp.SetRecorder(recorder)
currentLp.LogRequest(req)
}
@@ -227,6 +239,7 @@ func setupRouter(server *handlers.Server, pyProxy *httputil.ReverseProxy) *chi.M
r := chi.NewRouter()
r.Use(middleware.Logger)
r.Use(middleware.Recoverer)
r.Use(server.RecordMiddleware)
r.Get("/", server.HandleRoot)
r.Get("/health", server.HandleHealth)
+1
View File
@@ -1,2 +1,3 @@
certs/
default/
interactions/
+17
View File
@@ -0,0 +1,17 @@
[
{
"name": "IPv4",
"regexp": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$",
"replacement": "{ip}"
},
{
"name": "DeviceID",
"regexp": "^[A-F0-9]{12}$",
"replacement": "{deviceId}"
},
{
"name": "AccountID",
"regexp": "^\\d{1,10}$",
"replacement": "{accountId}"
}
]
+1
View File
@@ -35,6 +35,7 @@ func (s *Server) HandleProxyRequest(w http.ResponseWriter, r *http.Request) {
lp := proxy.NewLoggingProxy(target.String(), s.proxyRedact)
lp.LogBody = s.proxyLogBody
lp.SetRecorder(s.recorder)
proxy := httputil.NewSingleHostReverseProxy(target)
// Update director to set the correct host and path
@@ -0,0 +1,93 @@
package handlers
import (
"bufio"
"bytes"
"fmt"
"io"
"net"
"net/http"
)
// RecordMiddleware returns a middleware that records "self" requests and responses.
func (s *Server) RecordMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if s.recorder == nil {
next.ServeHTTP(w, r)
return
}
// Buffer the request body if it exists
var reqBody []byte
if r.Body != nil {
var err error
reqBody, err = io.ReadAll(r.Body)
if err == nil {
r.Body = io.NopCloser(bytes.NewBuffer(reqBody))
}
}
// wrap ResponseWriter to capture the response
rw := &responseWriter{
ResponseWriter: w,
body: &bytes.Buffer{},
}
next.ServeHTTP(rw, r)
// Create a response object for the recorder
res := rw.getRecordedResponse(r)
// Put back the original request body for recording
r.Body = io.NopCloser(bytes.NewBuffer(reqBody))
_ = s.recorder.Record("self", r, res)
})
}
type responseWriter struct {
http.ResponseWriter
statusCode int
body *bytes.Buffer
}
func (rw *responseWriter) Header() http.Header {
return rw.ResponseWriter.Header()
}
func (rw *responseWriter) WriteHeader(code int) {
rw.statusCode = code
rw.ResponseWriter.WriteHeader(code)
}
func (rw *responseWriter) Write(b []byte) (int, error) {
rw.body.Write(b)
return rw.ResponseWriter.Write(b)
}
func (rw *responseWriter) getRecordedResponse(r *http.Request) *http.Response {
statusCode := rw.statusCode
if statusCode == 0 {
statusCode = http.StatusOK
}
return &http.Response{
StatusCode: statusCode,
Header: rw.ResponseWriter.Header(),
Body: io.NopCloser(bytes.NewBuffer(rw.body.Bytes())),
Request: r,
}
}
func (rw *responseWriter) Flush() {
if f, ok := rw.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
if h, ok := rw.ResponseWriter.(http.Hijacker); ok {
return h.Hijack()
}
return nil, nil, fmt.Errorf("ResponseWriter does not support Hijacker")
}
+7
View File
@@ -8,6 +8,7 @@ import (
"github.com/gesellix/bose-soundtouch/pkg/discovery"
"github.com/gesellix/bose-soundtouch/pkg/models"
"github.com/gesellix/bose-soundtouch/pkg/service/datastore"
"github.com/gesellix/bose-soundtouch/pkg/service/proxy"
"github.com/gesellix/bose-soundtouch/pkg/service/setup"
)
@@ -20,6 +21,7 @@ type Server struct {
discovering bool
proxyRedact bool
proxyLogBody bool
recorder *proxy.Recorder
}
// NewServer creates a new SoundTouch service server.
@@ -34,6 +36,11 @@ func NewServer(ds *datastore.DataStore, sm *setup.Manager, serverURL string, pro
}
}
// SetRecorder sets the recorder for the server.
func (s *Server) SetRecorder(r *proxy.Recorder) {
s.recorder = r
}
// DiscoverDevices starts a background device discovery process.
//
//nolint:contextcheck
+67
View File
@@ -0,0 +1,67 @@
package proxy
import (
"encoding/json"
"fmt"
"os"
"regexp"
)
// PathPattern defines a regex and its replacement for sanitizing URL paths.
type PathPattern struct {
Name string `json:"name"`
Regexp string `json:"regexp"`
Replacement string `json:"replacement"`
compiled *regexp.Regexp
}
// PathPatterns is a collection of PathPattern.
type PathPatterns []PathPattern
// LoadPatterns loads path patterns from a JSON file.
func LoadPatterns(path string) (PathPatterns, error) {
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return PathPatterns{}, nil
}
return nil, err
}
var patterns PathPatterns
if err := json.Unmarshal(data, &patterns); err != nil {
return nil, err
}
for i := range patterns {
re, err := regexp.Compile(patterns[i].Regexp)
if err != nil {
return nil, fmt.Errorf("invalid regex in pattern %s: %w", patterns[i].Name, err)
}
patterns[i].compiled = re
}
return patterns, nil
}
// Sanitize sanitizes a segment using the configured patterns.
func (pp PathPatterns) Sanitize(segment string) (string, string) {
for _, p := range pp {
if p.compiled != nil && p.compiled.MatchString(segment) {
return p.Replacement, p.Replacement
}
}
return segment, ""
}
// DefaultPatterns returns the default set of path patterns.
func DefaultPatterns() PathPatterns {
p := PathPattern{
Name: "IPv4",
Regexp: `^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$`,
Replacement: "{ip}",
}
re, _ := regexp.Compile(p.Regexp)
p.compiled = re
return PathPatterns{p}
}
+10
View File
@@ -24,6 +24,7 @@ type LoggingProxy struct {
Redact bool
LogBody bool
MaxBodySize int64
Recorder *Recorder
}
// NewLoggingProxy creates a lightweight logger for HTTP requests/responses.
@@ -36,6 +37,11 @@ func NewLoggingProxy(_ string, redact bool) *LoggingProxy {
}
}
// SetRecorder sets the recorder for the proxy.
func (lp *LoggingProxy) SetRecorder(r *Recorder) {
lp.Recorder = r
}
// LogRequest prints an abbreviated request with optional header/body redaction.
func (lp *LoggingProxy) LogRequest(r *http.Request) {
headers := formatHeaders(r.Header, lp.Redact)
@@ -82,6 +88,10 @@ func (lp *LoggingProxy) LogResponse(r *http.Response) {
}
log.Printf("[PROXY_RES] %d %s\n Headers:\n%s\n Body: %s", r.StatusCode, r.Request.URL.String(), headers, bodyStr)
if lp.Recorder != nil {
_ = lp.Recorder.Record("upstream", r.Request, r)
}
}
func formatHeaders(h http.Header, redact bool) string {
+177
View File
@@ -0,0 +1,177 @@
package proxy
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
)
// Recorder handles persisting HTTP interactions as .http files.
type Recorder struct {
BaseDir string
SessionID string
SessionDir string
Patterns PathPatterns
counter uint64
variables map[string]string
mu sync.Mutex
}
// NewRecorder creates a new HTTP interaction recorder.
func NewRecorder(baseDir string) *Recorder {
sessionID := time.Now().Format("20060102-150405") + "-" + fmt.Sprintf("%d", os.Getpid())
return &Recorder{
BaseDir: baseDir,
SessionID: sessionID,
Patterns: DefaultPatterns(),
variables: make(map[string]string),
}
}
// Record persists a request and response to a .http file in the specified category (e.g., "self" or "upstream").
func (r *Recorder) Record(category string, req *http.Request, res *http.Response) error {
if r.BaseDir == "" {
return nil
}
// Group by URL path, sanitizing variable segments like IP addresses
pathSegments := strings.Split(strings.Trim(req.URL.Path, "/"), "/")
sanitizedSegments := make([]string, 0, len(pathSegments))
replacements := make(map[string]string)
for _, segment := range pathSegments {
if segment == "" {
continue
}
sanitized, replacement := r.Patterns.Sanitize(segment)
sanitizedSegments = append(sanitizedSegments, sanitized)
if replacement != "" {
replacements[segment] = replacement
}
}
subDir := "root"
if len(sanitizedSegments) > 0 {
subDir = filepath.Join(sanitizedSegments...)
}
dir := filepath.Join(r.BaseDir, "interactions", r.SessionID, category, subDir)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create directory %s: %w", dir, err)
}
timestamp := time.Now().Format("15-04-05.000")
count := atomic.AddUint64(&r.counter, 1)
filename := fmt.Sprintf("%04d-%s-%s.http", count, timestamp, req.Method)
path := filepath.Join(dir, filename)
var buf bytes.Buffer
// Write Request
displayURL := req.URL.String()
for orig, repl := range replacements {
displayURL = strings.ReplaceAll(displayURL, orig, "{{"+strings.Trim(repl, "{}")+"}}")
}
buf.WriteString(fmt.Sprintf("### %s %s\n", req.Method, displayURL))
buf.WriteString(fmt.Sprintf("%s %s\n", req.Method, displayURL))
for k, vv := range req.Header {
for _, v := range vv {
val := v
for orig, repl := range replacements {
val = strings.ReplaceAll(val, orig, "{{"+strings.Trim(repl, "{}")+"}}")
}
buf.WriteString(fmt.Sprintf("%s: %s\n", k, val))
}
}
buf.WriteString("\n")
if req.Body != nil {
bodyBytes, err := io.ReadAll(req.Body)
if err == nil {
req.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
buf.Write(bodyBytes)
buf.WriteString("\n")
}
}
// Write Response if available
if res != nil {
buf.WriteString("\n")
buf.WriteString("> {% \n")
buf.WriteString(fmt.Sprintf(" // Response: %d %s\n", res.StatusCode, http.StatusText(res.StatusCode)))
buf.WriteString(" // Headers:\n")
for k, vv := range res.Header {
for _, v := range vv {
buf.WriteString(fmt.Sprintf(" // %s: %s\n", k, v))
}
}
buf.WriteString("%}\n")
if res.Body != nil {
bodyBytes, err := io.ReadAll(res.Body)
if err == nil {
res.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
contentType := res.Header.Get("Content-Type")
if strings.Contains(contentType, "xml") || strings.Contains(contentType, "json") || strings.Contains(contentType, "text") {
buf.WriteString("\n/*\n")
buf.Write(bodyBytes)
buf.WriteString("\n*/\n")
} else {
buf.WriteString(fmt.Sprintf("\n// [Binary response body: %d bytes]\n", len(bodyBytes)))
}
}
}
}
if err := os.WriteFile(path, buf.Bytes(), 0644); err != nil {
return err
}
return r.updateEnvFile(replacements)
}
func (r *Recorder) updateEnvFile(newVars map[string]string) error {
if len(newVars) == 0 {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
changed := false
for orig, repl := range newVars {
key := strings.Trim(repl, "{}")
if r.variables[key] != orig {
r.variables[key] = orig
changed = true
}
}
if !changed {
return nil
}
envFile := filepath.Join(r.BaseDir, "interactions", r.SessionID, "http-client.env.json")
// Create the structure: {"session": {"key": "val"}}
content := map[string]map[string]string{
"session": r.variables,
}
data, err := json.MarshalIndent(content, "", " ")
if err != nil {
return err
}
return os.WriteFile(envFile, data, 0644)
}
+281
View File
@@ -0,0 +1,281 @@
package proxy
import (
"encoding/json"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
)
func TestRecorder_Record_Structure(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "recorder-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
r := NewRecorder(tmpDir)
tests := []struct {
name string
category string
path string
expected string // Expected subdirectory after interactions/{sessionID}/{category}/
}{
{
name: "root_path",
category: "self",
path: "/",
expected: "root",
},
{
name: "simple_path",
category: "self",
path: "/setup/info",
expected: "setup/info",
},
{
name: "path_with_ip",
category: "self",
path: "/setup/info/192.168.178.35",
expected: "setup/info/{ip}",
},
{
name: "upstream_path",
category: "upstream",
path: "/v1/playback/station/s123",
expected: "v1/playback/station/s123",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req := &http.Request{
Method: "GET",
URL: &url.URL{
Path: tt.path,
},
Header: make(http.Header),
}
err := r.Record(tt.category, req, nil)
if err != nil {
t.Fatalf("Record failed: %v", err)
}
expectedDir := filepath.Join(tmpDir, "interactions", r.SessionID, tt.category, tt.expected)
if _, err := os.Stat(expectedDir); os.IsNotExist(err) {
t.Errorf("Expected directory %s does not exist", expectedDir)
}
// Check if file was created
files, _ := os.ReadDir(expectedDir)
if len(files) == 0 {
t.Errorf("No files created in %s", expectedDir)
}
for _, f := range files {
if !strings.Contains(f.Name(), "-GET.http") {
t.Errorf("Unexpected filename: %s", f.Name())
}
// Verify prefix is 4 digits
if len(f.Name()) < 5 || !isDigit(f.Name()[0]) || !isDigit(f.Name()[1]) || !isDigit(f.Name()[2]) || !isDigit(f.Name()[3]) || f.Name()[4] != '-' {
t.Errorf("Filename %s does not have correct 0000- prefix", f.Name())
}
}
})
}
}
func TestRecorder_Record_Sanitization(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "recorder-sanitization-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
r := NewRecorder(tmpDir)
// Add a custom pattern
r.Patterns = append(r.Patterns, PathPattern{
Name: "DeviceID",
Regexp: `^A81B\w{8}$`,
Replacement: "{deviceId}",
})
// Re-compile
for i := range r.Patterns {
re, _ := regexp.Compile(r.Patterns[i].Regexp)
r.Patterns[i].compiled = re
}
req := &http.Request{
Method: "GET",
URL: &url.URL{
Path: "/info/192.168.178.35/A81B6A536A98",
},
Header: make(http.Header),
}
req.Header.Set("X-Device", "A81B6A536A98")
err = r.Record("self", req, nil)
if err != nil {
t.Fatalf("Record failed: %v", err)
}
expectedDir := filepath.Join(tmpDir, "interactions", r.SessionID, "self", "info", "{ip}", "{deviceId}")
if _, err := os.Stat(expectedDir); os.IsNotExist(err) {
t.Errorf("Expected directory %s does not exist", expectedDir)
}
files, _ := os.ReadDir(expectedDir)
if len(files) == 0 {
t.Fatalf("No files created in %s", expectedDir)
}
content, _ := os.ReadFile(filepath.Join(expectedDir, files[0].Name()))
contentStr := string(content)
if !strings.Contains(contentStr, "### GET /info/{{ip}}/{{deviceId}}") {
t.Errorf("Expected sanitized comment in .http file, got:\n%s", contentStr)
}
if !strings.Contains(contentStr, "GET /info/{{ip}}/{{deviceId}}") {
t.Errorf("Expected sanitized URL in .http file, got:\n%s", contentStr)
}
if !strings.Contains(contentStr, "X-Device: {{deviceId}}") {
t.Errorf("Expected sanitized Header in .http file, got:\n%s", contentStr)
}
}
func TestRecorder_Record_Sanitization_Account(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "recorder-sanitization-account-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
r := NewRecorder(tmpDir)
// Add AccountID pattern
r.Patterns = append(r.Patterns, PathPattern{
Name: "AccountID",
Regexp: `^\d{1,10}$`,
Replacement: "{accountId}",
})
// Re-compile
for i := range r.Patterns {
re, _ := regexp.Compile(r.Patterns[i].Regexp)
r.Patterns[i].compiled = re
}
req := &http.Request{
Method: "GET",
URL: &url.URL{
Path: "/marge/accounts/12345/full",
},
Header: make(http.Header),
}
err = r.Record("self", req, nil)
if err != nil {
t.Fatalf("Record failed: %v", err)
}
expectedDir := filepath.Join(tmpDir, "interactions", r.SessionID, "self", "marge", "accounts", "{accountId}", "full")
if _, err := os.Stat(expectedDir); os.IsNotExist(err) {
t.Errorf("Expected directory %s does not exist", expectedDir)
}
files, _ := os.ReadDir(expectedDir)
if len(files) == 0 {
t.Fatalf("No files created in %s", expectedDir)
}
content, _ := os.ReadFile(filepath.Join(expectedDir, files[0].Name()))
contentStr := string(content)
if !strings.Contains(contentStr, "### GET /marge/accounts/{{accountId}}/full") {
t.Errorf("Expected sanitized comment in .http file, got:\n%s", contentStr)
}
if !strings.Contains(contentStr, "GET /marge/accounts/{{accountId}}/full") {
t.Errorf("Expected sanitized URL in .http file, got:\n%s", contentStr)
}
}
func isDigit(c byte) bool {
return c >= '0' && c <= '9'
}
func TestRecorder_IncreasingPrefix(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "recorder-prefix-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
r := NewRecorder(tmpDir)
req := &http.Request{
Method: "GET",
URL: &url.URL{
Path: "/test",
},
Header: make(http.Header),
}
for i := 1; i <= 3; i++ {
err := r.Record("self", req, nil)
if err != nil {
t.Fatalf("Record failed: %v", err)
}
}
expectedDir := filepath.Join(tmpDir, "interactions", r.SessionID, "self", "test")
files, _ := os.ReadDir(expectedDir)
if len(files) != 3 {
t.Fatalf("Expected 3 files, got %d", len(files))
}
expectedPrefixes := []string{"0001-", "0002-", "0003-"}
for i, f := range files {
if !strings.HasPrefix(f.Name(), expectedPrefixes[i]) {
t.Errorf("File %d: expected prefix %s, got %s", i, expectedPrefixes[i], f.Name())
}
}
}
func TestRecorder_EnvFile(t *testing.T) {
tmpDir, err := os.MkdirTemp("", "recorder-env-test")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
r := NewRecorder(tmpDir)
req := &http.Request{
Method: "GET",
URL: &url.URL{
Path: "/info/192.168.178.35",
},
Header: make(http.Header),
}
err = r.Record("self", req, nil)
if err != nil {
t.Fatalf("Record failed: %v", err)
}
envFile := filepath.Join(tmpDir, "interactions", r.SessionID, "http-client.env.json")
if _, err := os.Stat(envFile); os.IsNotExist(err) {
t.Fatalf("Expected env file %s does not exist", envFile)
}
data, _ := os.ReadFile(envFile)
var content map[string]map[string]string
if err := json.Unmarshal(data, &content); err != nil {
t.Fatalf("Failed to unmarshal env file: %v", err)
}
if content["session"]["ip"] != "192.168.178.35" {
t.Errorf("Expected ip to be 192.168.178.35, got %s", content["session"]["ip"])
}
}