diff --git a/README.md b/README.md index d6b6488..6006827 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ A comprehensive solution for controlling and preserving Bose SoundTouch devices, - 🖥️ **CLI Tool**: Comprehensive command-line interface - 🌐 **SoundTouch Service**: Emulate Bose cloud services for offline device operation - 🔧 **Service Migration**: Migrate devices to use local services instead of Bose cloud -- 📊 **Traffic Analysis**: Proxy and log device communications for debugging +- 📊 **Traffic Analysis**: Proxy and log device communications +- 📝 **HTTP Recording**: Persist interactions as re-playable `.http` files - 🔒 **Production Ready**: Extensive testing with real SoundTouch hardware - 🌐 **Cross-Platform**: Windows, macOS, Linux support @@ -71,6 +72,7 @@ The `soundtouch-service` is a local server that emulates Bose's cloud services. - **🔧 Device Migration**: Seamlessly transition devices to local control - **🌐 Web Management UI**: Easy browser-based setup and management - **💾 Persistent Data**: Store presets, recents, and sources locally +- **📝 HTTP Recording**: Persist all interactions as re-playable `.http` files #### Quick Start: ```bash diff --git a/cmd/soundtouch-service/main.go b/cmd/soundtouch-service/main.go index af882a3..bf33701 100644 --- a/cmd/soundtouch-service/main.go +++ b/cmd/soundtouch-service/main.go @@ -5,6 +5,8 @@ package main import ( "context" "crypto/tls" + "flag" + "fmt" "log" "net/http" "net/http/httputil" @@ -31,6 +33,7 @@ func main() { server := handlers.NewServer(ds, sm, config.serverURL, config.redact, config.logBody) recorder := proxy.NewRecorder(config.dataDir) + recorder.Redact = config.redact patternsPath := filepath.Join(config.dataDir, "patterns.json") patterns, err := proxy.LoadPatterns(patternsPath) if err == nil && len(patterns) > 0 { @@ -75,29 +78,63 @@ type serviceConfig struct { } func loadConfig() serviceConfig { - port := os.Getenv("PORT") + // Define flags + fPort := flag.String("port", "", "Port to bind the service to (env: PORT)") + fBindAddr := flag.String("bind", "", "Network interface to bind to (env: BIND_ADDR)") + fTargetURL := flag.String("target-url", "", "URL for Python-based service components (env: PYTHON_BACKEND_URL)") + fDataDir := flag.String("data-dir", "", "Directory for persistent data (env: DATA_DIR)") + fServerURL := flag.String("server-url", "", "External URL of this service (env: SERVER_URL)") + fHttpsPort := flag.String("https-port", "", "HTTPS port to bind the service to (env: HTTPS_PORT)") + fHttpsServerURL := flag.String("https-server-url", "", "External HTTPS URL (env: HTTPS_SERVER_URL)") + fRedact := flag.String("redact-logs", "", "Redact sensitive data in proxy logs (true/false, env: REDACT_PROXY_LOGS)") + fLogBody := flag.String("log-bodies", "", "Log full request/response bodies (true/false, env: LOG_PROXY_BODY)") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage of soundtouch-service:\n") + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "\nConfiguration can also be set via environment variables.\n") + } + + flag.Parse() + + port := *fPort + if port == "" { + port = os.Getenv("PORT") + } if port == "" { port = "8000" } - bindAddr := os.Getenv("BIND_ADDR") + bindAddr := *fBindAddr + if bindAddr == "" { + bindAddr = os.Getenv("BIND_ADDR") + } addr := bindAddr + ":" + port if bindAddr == "" { addr = ":" + port } - targetURL := os.Getenv("PYTHON_BACKEND_URL") + targetURL := *fTargetURL + if targetURL == "" { + targetURL = os.Getenv("PYTHON_BACKEND_URL") + } if targetURL == "" { targetURL = "http://localhost:8001" } - dataDir := os.Getenv("DATA_DIR") + dataDir := *fDataDir + if dataDir == "" { + dataDir = os.Getenv("DATA_DIR") + } if dataDir == "" { dataDir = "data" } - serverURL := os.Getenv("SERVER_URL") + serverURL := *fServerURL + if serverURL == "" { + serverURL = os.Getenv("SERVER_URL") + } if serverURL == "" { hostname, _ := os.Hostname() if hostname == "" { @@ -107,7 +144,10 @@ func loadConfig() serviceConfig { serverURL = "http://" + strings.ToLower(hostname) + ":" + port } - httpsPort := os.Getenv("HTTPS_PORT") + httpsPort := *fHttpsPort + if httpsPort == "" { + httpsPort = os.Getenv("HTTPS_PORT") + } if httpsPort == "" { httpsPort = "8443" } @@ -117,7 +157,10 @@ func loadConfig() serviceConfig { httpsAddr = ":" + httpsPort } - httpsServerURL := os.Getenv("HTTPS_SERVER_URL") + httpsServerURL := *fHttpsServerURL + if httpsServerURL == "" { + httpsServerURL = os.Getenv("HTTPS_SERVER_URL") + } if httpsServerURL == "" { hostname, _ := os.Hostname() if hostname == "" { @@ -159,6 +202,18 @@ func loadConfig() serviceConfig { domains = append(domains, d) } + redactVal := *fRedact + if redactVal == "" { + redactVal = os.Getenv("REDACT_PROXY_LOGS") + } + redact := redactVal != "false" + + logBodyVal := *fLogBody + if logBodyVal == "" { + logBodyVal = os.Getenv("LOG_PROXY_BODY") + } + logBody := logBodyVal == "true" + return serviceConfig{ port: port, bindAddr: bindAddr, @@ -168,8 +223,8 @@ func loadConfig() serviceConfig { serverURL: serverURL, httpsServerURL: httpsServerURL, httpsAddr: httpsAddr, - redact: os.Getenv("REDACT_PROXY_LOGS") != "false", - logBody: os.Getenv("LOG_PROXY_BODY") == "true", + redact: redact, + logBody: logBody, domains: domains, } } diff --git a/docs/SOUNDTOUCH-SERVICE.md b/docs/SOUNDTOUCH-SERVICE.md index 4cde02e..2c6ca66 100644 --- a/docs/SOUNDTOUCH-SERVICE.md +++ b/docs/SOUNDTOUCH-SERVICE.md @@ -11,6 +11,7 @@ The service provides: - **📊 Traffic Proxying**: Inspect and log all device communications for debugging - **🌐 Web Management UI**: Browser-based interface for device management - **💾 Persistent Data**: Store device configurations, presets, and usage statistics +- **📝 HTTP Recording**: Persist all interactions as re-playable `.http` files - **🔍 Auto-Discovery**: Automatically detect and configure SoundTouch devices - **🔒 Offline Operation**: Continue using full device functionality without internet @@ -365,6 +366,48 @@ The web management interface provides a comprehensive dashboard for managing you 3. **Troubleshooting**: Use the debug tools to diagnose device connectivity issues 4. **Log Analysis**: Enable detailed logging for development and troubleshooting +## HTTP Interaction Recording + +The service automatically records all HTTP interactions (both those handled locally and those proxied upstream) as `.http` files. These files are compatible with the [IntelliJ IDEA HTTP Client](https://www.jetbrains.com/help/idea/exploring-http-syntax.html). + +### Key Features + +- **Session Grouping**: All interactions from a single server session are stored in a dedicated directory named `{timestamp}-{pid}`. +- **Chronological Order**: Files are prefixed with a sequential number (e.g., `0001-`, `0002-`) to preserve the exact order of requests across the entire session. +- **Path-Based Structure**: Recordings are organized into subdirectories based on their URL path for better discoverability. +- **Automatic Sanitization**: Variable path segments like IP addresses, Device IDs, and Account IDs are automatically identified and replaced with placeholders (e.g., `{{ip}}`, `{{deviceId}}`). The original values are preserved as comments at the top of the recorded `.http` files for easy identification. +- **Re-playability**: An `http-client.env.json` file is generated for each session, allowing you to re-play the recorded requests immediately in IntelliJ IDEA. + +### Configuration + +#### Redaction + +By default, the service redacts sensitive information from the recorded `.http` files, including: +- `Authorization` headers +- `Cookie` headers +- `X-Bose-Token` headers + +This behavior is controlled by the `--redact-logs` flag or the `REDACT_PROXY_LOGS` environment variable. + +#### Custom Patterns + +The service uses regex patterns to identify variable segments in URL paths. These patterns are loaded from `data/patterns.json`. You can add custom patterns to this file to support additional variable segments: + +```json +[ + { + "name": "MyVariable", + "regexp": "^[0-9]{5}$", + "replacement": "{myVar}" + } +] +``` + +Variables found via these patterns will be: +1. Used as directory names in the `interactions/` folder. +2. Parameterized as `{{myVar}}` within the `.http` files. +3. Added to the `http-client.env.json` file with their actual values. + ## Persistent Data ### Data Directory Structure @@ -383,6 +426,15 @@ data/ │ ├── Sources.xml │ ├── Presets.xml │ └── Recents.xml +├── interactions/ +│ └── {SESSION_ID}/ +│ ├── self/ +│ │ └── {PATH}/ +│ │ └── {SEQ}-{TIME}-{METHOD}.http +│ ├── upstream/ +│ │ └── {PATH}/ +│ │ └── {SEQ}-{TIME}-{METHOD}.http +│ └── http-client.env.json ├── stats/ │ ├── usage/ │ │ └── *.json @@ -411,6 +463,14 @@ data/ #### Events (`events/`) - **device_events_*.log**: Device event history and debugging logs +#### HTTP Interactions (`interactions/`) +- **{SESSION_ID}/**: A unique directory per server run (format: `YYYYMMDD-HHMMSS-PID`). +- **self/**: Requests handled directly by the service. +- **upstream/**: Requests proxied to external Bose services. +- **{PATH}/**: Nested subdirectories reflecting the URL path (sanitized). +- **http-client.env.json**: IntelliJ IDEA HTTP Client environment file with session variables. +- **{SEQ}-{TIME}-{METHOD}.http**: Individual interaction recordings in standard HTTP Client format. + ### Data Management #### Backup Strategy diff --git a/pkg/service/proxy/recorder.go b/pkg/service/proxy/recorder.go index 44caf9e..4b7a966 100644 --- a/pkg/service/proxy/recorder.go +++ b/pkg/service/proxy/recorder.go @@ -20,6 +20,7 @@ type Recorder struct { SessionID string SessionDir string Patterns PathPatterns + Redact bool counter uint64 variables map[string]string mu sync.Mutex @@ -82,8 +83,16 @@ func (r *Recorder) Record(category string, req *http.Request, res *http.Response } buf.WriteString(fmt.Sprintf("### %s %s\n", req.Method, displayURL)) + for orig, repl := range replacements { + key := strings.Trim(repl, "{}") + buf.WriteString(fmt.Sprintf("// %s: %s\n", key, orig)) + } buf.WriteString(fmt.Sprintf("%s %s\n", req.Method, displayURL)) for k, vv := range req.Header { + if r.Redact && isSensitive(k) { + buf.WriteString(fmt.Sprintf("%s: [REDACTED]\n", k)) + continue + } for _, v := range vv { val := v for orig, repl := range replacements { @@ -110,6 +119,10 @@ func (r *Recorder) Record(category string, req *http.Request, res *http.Response buf.WriteString(fmt.Sprintf(" // Response: %d %s\n", res.StatusCode, http.StatusText(res.StatusCode))) buf.WriteString(" // Headers:\n") for k, vv := range res.Header { + if r.Redact && isSensitive(k) { + buf.WriteString(fmt.Sprintf(" // %s: [REDACTED]\n", k)) + continue + } for _, v := range vv { buf.WriteString(fmt.Sprintf(" // %s: %s\n", k, v)) } diff --git a/pkg/service/proxy/recorder_test.go b/pkg/service/proxy/recorder_test.go index 2985e9e..29e056c 100644 --- a/pkg/service/proxy/recorder_test.go +++ b/pkg/service/proxy/recorder_test.go @@ -200,6 +200,54 @@ func TestRecorder_Record_Sanitization_Account(t *testing.T) { if !strings.Contains(contentStr, "GET /marge/accounts/{{accountId}}/full") { t.Errorf("Expected sanitized URL in .http file, got:\n%s", contentStr) } + if !strings.Contains(contentStr, "// accountId: 12345") { + t.Errorf("Expected accountId comment in .http file, got:\n%s", contentStr) + } +} + +func TestRecorder_Record_Redaction(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "recorder-redaction-test") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + r := NewRecorder(tmpDir) + r.Redact = true + + req := &http.Request{ + Method: "GET", + URL: &url.URL{ + Path: "/test", + }, + Header: make(http.Header), + } + req.Header.Set("Authorization", "Bearer sensitive-token") + req.Header.Set("Cookie", "session=secret") + req.Header.Set("X-Normal", "public-info") + + err = r.Record("self", req, nil) + if err != nil { + t.Fatalf("Record failed: %v", err) + } + + expectedDir := filepath.Join(tmpDir, "interactions", r.SessionID, "self", "test") + files, _ := os.ReadDir(expectedDir) + content, _ := os.ReadFile(filepath.Join(expectedDir, files[0].Name())) + contentStr := string(content) + + if !strings.Contains(contentStr, "Authorization: [REDACTED]") { + t.Errorf("Expected Authorization header to be redacted, got:\n%s", contentStr) + } + if strings.Contains(contentStr, "sensitive-token") { + t.Errorf("Sensitive token still present in content:\n%s", contentStr) + } + if !strings.Contains(contentStr, "Cookie: [REDACTED]") { + t.Errorf("Expected Cookie header to be redacted, got:\n%s", contentStr) + } + if !strings.Contains(contentStr, "X-Normal: public-info") { + t.Errorf("Expected normal header to be present, got:\n%s", contentStr) + } } func isDigit(c byte) bool {