From 747a9cec979f3408dfaa2f86fbbbd083fee422cd Mon Sep 17 00:00:00 2001 From: Tobias Gesellchen Date: Sun, 19 Apr 2026 22:27:54 +0200 Subject: [PATCH] Add app analyzing/debugging docs and scripts (#174) --- .github/workflows/ci.yml | 9 + .github/workflows/release.yml | 3 + .github/workflows/security.yml | 9 + docs/SUMMARY.md | 1 + docs/analysis/BOSE-APP-ADB-Emulator.md | 341 +++++++++++++++++++++++ docs/analysis/BOSE-LAB-RUNBOOK.md | 161 +++++++++-- go.mod | 2 + go.sum | 27 ++ scripts/extract-log-interactions/main.go | 108 +++++++ scripts/extract-ws/main.go | 254 +++++++++++++++++ scripts/http-diff/README.md | 32 +++ scripts/http-diff/main.go | 299 ++++++++++++++++++++ 12 files changed, 1223 insertions(+), 23 deletions(-) create mode 100644 docs/analysis/BOSE-APP-ADB-Emulator.md create mode 100644 scripts/extract-log-interactions/main.go create mode 100644 scripts/extract-ws/main.go create mode 100644 scripts/http-diff/README.md create mode 100644 scripts/http-diff/main.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e59eb12..94664e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: restore-keys: | ${{ runner.os }}-go- + - name: Install libpcap + run: sudo apt-get install -y libpcap-dev + - name: Download dependencies run: go mod download @@ -70,6 +73,9 @@ jobs: with: go-version-file: "go.mod" + - name: Install libpcap + run: sudo apt-get install -y libpcap-dev + - name: Run golangci-lint uses: golangci/golangci-lint-action@v9 with: @@ -127,6 +133,9 @@ jobs: with: go-version-file: "go.mod" + - name: Install libpcap + run: sudo apt-get install -y libpcap-dev + - name: Run basic vulnerability check run: | go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c722801..d8f5573 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,6 +68,9 @@ jobs: with: go-version-file: ${{ env.GO_VERSION_FILE }} + - name: Install libpcap + run: sudo apt-get install -y libpcap-dev + - name: Run tests before release run: | echo "Running final tests before release..." diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index cde37ae..4b7de8c 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -26,6 +26,9 @@ jobs: with: go-version-file: "go.mod" + - name: Install libpcap + run: sudo apt-get install -y libpcap-dev + - name: Install security scanning tools run: | go install golang.org/x/vuln/cmd/govulncheck@latest @@ -67,6 +70,9 @@ jobs: with: go-version-file: "go.mod" + - name: Install libpcap + run: sudo apt-get install -y libpcap-dev + - name: Install static analysis tools run: | go install honnef.co/go/tools/cmd/staticcheck@latest @@ -106,6 +112,9 @@ jobs: - name: Checkout code uses: actions/checkout@v6 + - name: Install libpcap + run: sudo apt-get install -y libpcap-dev + - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 5ba2c7b..ad82f5f 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -60,6 +60,7 @@ * [IoT Configuration Analysis](analysis/IOT-CONFIGURATION-ANALYSIS.md) * [Bose Lab Runbook](analysis/BOSE-LAB-RUNBOOK.md) * [Missing Routes Spotify](analysis/MISSING-ROUTES-SPOTIFY.md) +* [Bose App ADB Emulator](analysis/BOSE-APP-ADB-Emulator.md) ## Parity Analysis * [Parity Improvements](PARITY-IMPROVEMENTS.md) diff --git a/docs/analysis/BOSE-APP-ADB-Emulator.md b/docs/analysis/BOSE-APP-ADB-Emulator.md new file mode 100644 index 0000000..abdd615 --- /dev/null +++ b/docs/analysis/BOSE-APP-ADB-Emulator.md @@ -0,0 +1,341 @@ +# Bose SoundTouch Traffic Interception Runbook + +Intercept HTTPS/WebSocket traffic from the Bose SoundTouch Android app using an Android emulator, mitmproxy, and Frida. Tested on Apple Silicon (ARM64) Mac. + +## Prerequisites + +- Android Studio installed (for SDK tools and emulator) +- Docker installed +- mitmproxy installed (`pip install mitmproxy` or via your preferred method) +- The Bose SoundTouch APK (extracted from a real device, see below) + +Add Android SDK tools to your PATH (add to `~/.zshrc`): + +```bash +export PATH=$PATH:~/Library/Android/sdk/emulator +export PATH=$PATH:~/Library/Android/sdk/platform-tools +``` + +--- + +## 1. Extract APK from Real Device + +Connect your Android device via USB with USB debugging enabled. + +```bash +adb devices +# note your device ID, e.g. "ABC123" + +adb -s ABC123 shell pm path com.bose.soundtouch +# output e.g.: package:/data/app/~~xyz/com.bose.soundtouch-abc/base.apk + +adb -s ABC123 pull /data/app/~~xyz/com.bose.soundtouch-abc/base.apk bose.apk +``` + +--- + +## 2. Create Android Emulator (ARM64, API 33) + +On Apple Silicon you need an ARM64 image. Use the `avdmanager` and `sdkmanager` CLI tools. + +```bash +# Install the system image +~/Library/Android/sdk/cmdline-tools/latest/bin/sdkmanager \ + "system-images;android-33;google_apis;arm64-v8a" + +# Create the AVD +~/Library/Android/sdk/cmdline-tools/latest/bin/avdmanager create avd \ + -n Pixel_6_API33 \ + -k "system-images;android-33;google_apis;arm64-v8a" \ + -d "pixel_6" +``` + +Alternatively create the AVD via Android Studio Device Manager (choose "Google APIs", arm64-v8a, API 33). + +--- + +## 3. Start Emulator with Writable System + +```bash +# List available AVDs +~/Library/Android/sdk/emulator/emulator -list-avds + +# Start with writable system partition +~/Library/Android/sdk/emulator/emulator -avd Pixel_6_API33 -writable-system +``` + +Wait until the emulator has fully booted, then: + +```bash +adb -s emulator-5554 root +adb -s emulator-5554 shell avbctl disable-verification +adb -s emulator-5554 reboot + +# After reboot: +adb -s emulator-5554 root +``` + +--- + +## 4. Install Bose APK + +```bash +adb -s emulator-5554 install bose.apk +``` + +--- + +## 5. Set Up mitmproxy + +```bash +# Start mitmproxy (generates CA cert on first run) +mitmweb --port 8080 --mode regular -w bose_traffic.mitm +``` + +Extract the CA certificate (without private key): + +```bash +openssl x509 -in ~/.mitmproxy/mitmproxy-ca.pem -out ~/.mitmproxy/mitmproxy-ca-cert.pem + +# Verify it's the mitmproxy cert, not another cert: +openssl x509 -in ~/.mitmproxy/mitmproxy-ca-cert.pem -noout -issuer +# should show: issuer= /CN=mitmproxy/O=mitmproxy +``` + +--- + +## 6. Install mitmproxy CA Certificate in Emulator + +```bash +HASH=$(openssl x509 -inform PEM -subject_hash_old \ + -in ~/.mitmproxy/mitmproxy-ca-cert.pem | head -1) + +adb -s emulator-5554 push ~/.mitmproxy/mitmproxy-ca-cert.pem /data/local/tmp/mitmproxy.pem + +adb -s emulator-5554 shell su 0 mkdir -p /data/misc/user/0/cacerts-added + +adb -s emulator-5554 shell su 0 \ + cp /data/local/tmp/mitmproxy.pem /data/misc/user/0/cacerts-added/${HASH}.0 + +adb -s emulator-5554 shell su 0 \ + chmod 644 /data/misc/user/0/cacerts-added/${HASH}.0 +``` + +--- + +## 7. Set System Proxy in Emulator + +Find your Mac's local IP: + +```bash +ipconfig getifaddr en0 +# e.g. 192.168.1.123 +``` + +Set the proxy: + +```bash +adb -s emulator-5554 shell settings put global http_proxy 192.168.1.123:8080 +``` + +--- + +## 8. Set Up Frida (via Python venv) + +```bash +python3 -m venv /tmp/frida-venv +/tmp/frida-venv/bin/pip install frida==17.9.1 frida-tools==14.8.1 +``` + +Download the frida-server binary for ARM64 Android: + +```bash +FRIDA_VERSION=17.9.1 + +curl -L "https://github.com/frida/frida/releases/download/${FRIDA_VERSION}/frida-server-${FRIDA_VERSION}-android-arm64.xz" \ + -o /tmp/frida-server.xz + +unxz /tmp/frida-server.xz +mv /tmp/frida-server-${FRIDA_VERSION}-android-arm64 /tmp/frida-server +``` + +Push to emulator and start: + +```bash +adb -s emulator-5554 push /tmp/frida-server /data/local/tmp/frida-server +adb -s emulator-5554 shell su 0 chmod 755 /data/local/tmp/frida-server +adb -s emulator-5554 shell su 0 /data/local/tmp/frida-server & +``` + +--- + +## 9. Download SSL Bypass Scripts + +```bash +BASE=https://raw.githubusercontent.com/httptoolkit/frida-interception-and-unpinning/main + +curl -L "${BASE}/config.js" -o /tmp/config.js +curl -L "${BASE}/android/android-system-certificate-injection.js" \ + -o /tmp/android-system-certificate-injection.js +curl -L "${BASE}/android/android-proxy-override.js" \ + -o /tmp/android-proxy-override.js +curl -L "${BASE}/android/android-certificate-unpinning.js" \ + -o /tmp/android-certificate-unpinning.js +curl -L "${BASE}/android/android-certificate-unpinning-fallback.js" \ + -o /tmp/android-certificate-unpinning-fallback.js +``` + +--- + +## 10. Configure config.js + +Edit `/tmp/config.js` and set: + +```javascript +const CERT_PEM = ``; + +const PROXY_HOST = '192.168.1.123'; // your Mac IP +const PROXY_PORT = 8080; +``` + +Insert the full PEM content (from `-----BEGIN CERTIFICATE-----` to `-----END CERTIFICATE-----`) between the backticks. + +Quick check that the right cert is in place: + +```bash +# The issuer inside config.js should be mitmproxy, not SoundTouch +grep -A3 "CERT_PEM" /tmp/config.js | head -5 +``` + +--- + +## 11. Start Interception + +Make sure mitmweb is running, then: + +```bash +/tmp/frida-venv/bin/frida \ + -U \ + -f com.bose.soundtouch \ + -l /tmp/config.js \ + -l /tmp/android-system-certificate-injection.js \ + -l /tmp/android-proxy-override.js \ + -l /tmp/android-certificate-unpinning.js \ + -l /tmp/android-certificate-unpinning-fallback.js +``` + +Expected output in the Frida REPL: + +``` +== System certificate trust injected == +== Proxy system configuration overridden to 192.168.1.123:8080 == +== Proxy configuration overridden to 192.168.1.123:8080 == +== Certificate unpinning completed == +== Unpinning fallback auto-patcher installed == +``` + +Open mitmweb at `http://127.0.0.1:8081` to observe traffic live. + +--- + +## 12. Save & Replay Recordings + +Traffic is saved to `bose_traffic.mitm` (set via `-w` flag in step 5). + +```bash +# Replay/analyse a saved recording: +mitmweb -r bose_traffic.mitm +``` + +--- + +## Cleanup + +```bash +# Remove proxy setting from emulator +adb -s emulator-5554 shell settings delete global http_proxy + +# Remove venv +rm -rf /tmp/frida-venv /tmp/frida-server /tmp/frida-server.xz +rm /tmp/config.js /tmp/android-*.js + +# Stop emulator +adb -s emulator-5554 emu kill +``` + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|-----------------------------------------|--------------------------------------------------|--------------------------------------------------------------------------------| +| `remount failed` | ARM64 emulator doesn't support overlayfs remount | Use `/data/misc/user/0/cacerts-added/` method instead | +| `TLS: Trust anchor not found` | Wrong certificate in config.js | Check issuer: must be mitmproxy, not SoundTouch | +| `Chain validation failed` | Private key included in cert | Re-extract with `openssl x509 -in mitmproxy-ca.pem -out mitmproxy-ca-cert.pem` | +| `frida-server: connection refused` | frida-server not running | Re-run `adb shell su 0 /data/local/tmp/frida-server &` | +| frida and frida-server version mismatch | Versions must be identical | Pin both to same version (e.g. `17.9.1`) | +| `emulator: multiple AVDs` error | Emulator already running | Kill first: `adb emu kill`, then restart with `-writable-system` | + +--- + +## App Automation Options + +For most traffic-recording purposes, manually operating the app while mitmproxy captures is sufficient. If you need to automate specific interactions (e.g. to repeatably capture the requests triggered by startup or a particular action), the following tools are available. + +### Starting the App + +```bash +# Via app drawer: swipe up on the home screen and tap "Bose SoundTouch" + +# Via adb monkey (simplest) +adb -s emulator-5554 shell monkey -p com.bose.soundtouch 1 + +# Via explicit intent (if the activity name is known) +adb -s emulator-5554 shell am start -n com.bose.soundtouch/.MainActivity + +# Look up all activities if the name is unknown +adb -s emulator-5554 shell dumpsys package com.bose.soundtouch | grep Activity +``` + +### adb — sufficient for simple cases + +```bash +# Tap at screen coordinates +adb shell input tap 540 960 + +# Swipe +adb shell input swipe 540 1500 540 500 + +# Type text +adb shell input text "mytext" + +# Take a screenshot +adb shell screencap /sdcard/screen.png && adb pull /sdcard/screen.png +``` + +### UIAutomator2 — inspect UI elements + +```bash +# Dump the current UI hierarchy to find element IDs +adb shell uiautomator dump /sdcard/ui.xml +adb pull /sdcard/ui.xml +``` + +Open `ui.xml` to find element resource IDs, then target them precisely in scripts. + +### Appium — full scripted automation + +```python +from appium import webdriver + +driver = webdriver.Remote('http://localhost:4723/wd/hub', { + 'platformName': 'Android', + 'appPackage': 'com.bose.soundtouch', + 'appActivity': '.MainActivity', +}) + +# Find an element by resource ID and tap it +driver.find_element('id', 'com.bose.soundtouch:id/play_button').click() +``` + +> **Note:** `monkey` is a stress-test tool that sends random events — use it only to launch the app, not to drive specific interactions. diff --git a/docs/analysis/BOSE-LAB-RUNBOOK.md b/docs/analysis/BOSE-LAB-RUNBOOK.md index 36c4cba..77d7837 100644 --- a/docs/analysis/BOSE-LAB-RUNBOOK.md +++ b/docs/analysis/BOSE-LAB-RUNBOOK.md @@ -585,7 +585,72 @@ adb install Bose-SoundTouch-patched.apk 2. On the phone, use a File Manager to open the APK. 3. If prompted, allow "Install from Unknown Sources" for your File Manager. -### Option C: Patching the App with Frida (Requires Root) +### Option C: Using the macOS Bose SoundTouch App (No Root/Patching Required) + +If you have a Mac, using the macOS version of the Bose SoundTouch app is often a good alternative. However, because the app is built on an **older version of Qt (5.7.0)**, it has specific trust and TLS compatibility issues that require extra steps. + +#### 1. Install the Custom CA in macOS Keychain + +1. Open **Keychain Access** on your Mac. +2. Select the **System** keychain (or **login** if System is locked). +3. Drag and drop your `ca.crt` file into the list. +4. Double-click the newly added certificate (e.g., "Bose-Lab Root CA"). +5. Expand the **Trust** section. +6. Set "When using this certificate" to **Always Trust**. +7. Close the window and authenticate with your Mac password. + +#### 2. Configure the Proxy + +You can either configure the macOS system proxy manually or use `mitmproxy`'s automatic interception. + +**Method 1: System Proxy (Manual)** +1. Go to **System Settings → Network → Wi-Fi → Details... → Proxies**. +2. Enable **HTTP Proxy** and **HTTPS Proxy**. +3. Set Server to your Pi's IP (`192.168.10.1`) and Port to `8080`. +4. Click **OK** and **Apply**. + +**Method 2: mitmproxy Local Redirect (Automatic)** +If you are running `mitmproxy` directly on your Mac (instead of the Pi), you can use the modern "Local Redirect" mode which doesn't require proxy settings: +```bash +# Install mitmproxy via Homebrew +brew install mitmproxy + +# Start mitmproxy in local redirect mode +# This uses a macOS Network Extension to intercept traffic from specific apps +mitmproxy --mode local +``` + +#### 3. Special Troubleshooting: Legacy Qt 5.7.0 SSL Failures + +If you see `SSL handshake failed` in the `mitmproxy` logs or the app's internal log (`log.txt`), the app's older networking stack is rejecting the connection. This is common because Qt 5.7.0 (2016) lacks support for **TLS 1.3** and many modern root certificates (like Let's Encrypt's **ISRG Root X1**). + +**The Solution: Launch with SSL Bypass Flags** + +Since the Bose macOS app is a hybrid of **Qt/Chromium** and **Node.js**, you must bypass the trust checks for both engines by launching the app from the terminal: + +```bash +# 1. Bypass QtWebEngine/Chromium (Qt 5.7) trust +export QTWEBENGINE_CHROMIUM_FLAGS="--ignore-certificate-errors" + +# 2. Bypass Node.js (SoundTouch Music Server) trust +export NODE_TLS_REJECT_UNAUTHORIZED=0 + +# 3. (Optional) Provide your custom CA directly to Node.js +export NODE_EXTRA_CA_CERTS="/path/to/your/ca.crt" + +# 4. Launch the application +"/Applications/SoundTouch/SoundTouch.app/Contents/MacOS/SoundTouch" +``` + +#### 4. Verify and Capture + +1. Open Safari and visit `https://neverssl.com`. Verify the certificate is issued by your custom CA. +2. Launch the Bose app using the terminal command above. +3. Watch the traffic flow in `mitmproxy`. + +> **Note:** Even on macOS, **Certificate Pinning** is still possible if Bose implemented it specifically in the desktop app code. However, it is much less common on desktop apps than on mobile apps. If it works, you've saved yourself hours of Android patching! + +### Option D: Patching the App with Frida (Requires Root) If the app uses **Certificate Pinning** (hardcoded hashes), even moving the CA to the System store won't work. You must disable the pinning check in the app's code. @@ -625,38 +690,57 @@ mitmproxy --listen-port 8080 ## Step 13 – Extracting for soundtouch-service +You can extract interactions (especially unencrypted WebSockets on port 8090) from a `.pcap` and format them for use in `soundtouch-service`. + +### 1. Extract Traffic using Go + +A helper script is provided in `scripts/extract-ws.go`. It automatically detects, unmasks, and decompresses (GZIP) WebSocket frames, and also extracts DNS, MDNS, and SSDP traffic. + ```bash -# Which IPs did the phone receive? -cat /var/lib/misc/dnsmasq.leases +# Install dependencies +go get github.com/google/gopacket -# Is the access point active? -sudo systemctl status hostapd +# Run extraction (outputs multiple files: .ws.http, .dns.txt, .mdns.txt, .ssdp.txt) +# The results will be saved beside your .pcap file +go run scripts/extract-ws.go your_capture.pcap [filter_ip] -# Is dnsmasq active? -sudo systemctl status dnsmasq +# Example: Filter for a specific speaker's IP in WebSocket messages +go run scripts/extract-ws.go capture.pcap 192.168.100.1 +``` -# Check interfaces and IPs -ip addr show +### 2. Manual Extraction with tshark -# Check routing table -ip route show +If you only need a quick look at the payloads: -# Show active nftables rules -sudo nft list ruleset - -# All running tcpdump processes -pgrep -a tcpdump - -# Test the Pi's own DNS resolution -dig @127.0.0.1 -p 5353 global.api.bose.io - -# Check network connectivity from the phone (from the Pi) -ping 192.168.10.101 # Phone IP from dnsmasq.leases +```bash +# Extract all WebSocket text payloads +tshark -r your_capture.pcap -Y "websocket.payload.text" -T fields -e websocket.payload.text ``` --- -## Restart Sequence +## Step 14 – Extracting from Internal App Logs (macOS) + +If you are using the macOS app and cannot decrypt the cloud traffic due to pinning, you can still extract the JSON/XML messages from the app's internal communication log. + +A helper script is provided in `scripts/extract-log-interactions.go`. It parses the interleaved "Native" and "Network" calls to reconstruct the application's internal state and cloud requests. + +```bash +# Run extraction from the log file +# Outputs a chronological record of internal events and network URLs +go run scripts/extract-log-interactions.go path/to/log.txt > extracted-interactions.http +``` + +**What this shows:** +- **TO NETWORK:** The URLs the app is about to call (intercepted before encryption). +- **FROM NATIVE:** Data being returned from the OS or Cloud to the UI. +- **TO NATIVE:** Commands being sent from the UI to the underlying engines. + +This is a powerful "Plan B" when HTTPS decryption is blocked, as the app essentially logs its own decrypted data for you. + +--- + +## Helper Commands / Troubleshooting After a Pi reboot, everything should come up automatically. If not: @@ -775,3 +859,34 @@ sudo openssl x509 -req -in bose.csr -CA ca.crt -CAkey ca.key \ ### 3. Usage in your DNS/HTTPS Server Your custom server (e.g., a small Go or Python script) would then use `bose.crt` and `bose.key` to serve HTTPS traffic for those domains. + +## Appendix B – Helpful Commands + +```bash +# Which IPs did the phone receive? +cat /var/lib/misc/dnsmasq.leases + +# Is the access point active? +sudo systemctl status hostapd + +# Is dnsmasq active? +sudo systemctl status dnsmasq + +# Check interfaces and IPs +ip addr show + +# Check routing table +ip route show + +# Show active nftables rules +sudo nft list ruleset + +# All running tcpdump processes +pgrep -a tcpdump + +# Test the Pi's own DNS resolution +dig @127.0.0.1 -p 5353 global.api.bose.io + +# Check network connectivity from the phone (from the Pi) +ping 192.168.10.101 # Phone IP from dnsmasq.leases +``` diff --git a/go.mod b/go.mod index f81453d..5470f40 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.26.2 require ( github.com/go-chi/chi/v5 v5.2.5 + github.com/google/gopacket v1.1.19 github.com/gorilla/websocket v1.5.3 github.com/hashicorp/mdns v1.0.6 github.com/miekg/dns v1.1.72 github.com/russross/blackfriday/v2 v2.1.0 + github.com/sergi/go-diff v1.4.0 github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef github.com/urfave/cli/v2 v2.27.7 diff --git a/go.sum b/go.sum index 2fdf7cc..4c9dee4 100644 --- a/go.sum +++ b/go.sum @@ -1,28 +1,44 @@ github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/mdns v1.0.6 h1:SV8UcjnQ/+C7KeJ/QeVD/mdN2EmzYfcGfufcuzxfCLQ= github.com/hashicorp/mdns v1.0.6/go.mod h1:X4+yWh+upFECLOki1doUPaKpgNQII9gy4bUdCYKNhmM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE= github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q= github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= @@ -32,6 +48,8 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww= golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -40,6 +58,7 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -62,6 +81,7 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -101,6 +121,7 @@ golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= @@ -109,3 +130,9 @@ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxb golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/scripts/extract-log-interactions/main.go b/scripts/extract-log-interactions/main.go new file mode 100644 index 0000000..bd4042a --- /dev/null +++ b/scripts/extract-log-interactions/main.go @@ -0,0 +1,108 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "regexp" + "strings" +) + +type Interaction struct { + Type string `json:"type"` // ToNative, FromNative, ToNetwork + Payload string `json:"payload"` // Raw string + Parsed interface{} `json:"parsed"` // JSON if possible + Timestamp string `json:"timestamp"` // If available + ID string `json:"id"` // Internal ID if available +} + +func main() { + if len(os.Args) < 2 { + fmt.Println("Usage: go run extract-log-interactions.go ") + return + } + + logFile := os.Args[1] + file, err := os.Open(logFile) + if err != nil { + fmt.Printf("Error opening file: %v\n", err) + return + } + defer file.Close() + + // Regex patterns + toNativeRegex := regexp.MustCompile(`To Native : "(.*)"`) + fromNativeRegex := regexp.MustCompile(`From Native : "(.*)"`) + toNetworkRegex := regexp.MustCompile(`To Network "(.*)"`) + timestampRegex := regexp.MustCompile(`Js_Console_Msg: "(\d{2}:\d{2}:\d{2}\.\d{3})`) + + scanner := bufio.NewScanner(file) + lastTimestamp := "" + + fmt.Println("### Bose SoundTouch Internal Log Interactions") + fmt.Println("-------------------------------------------------") + + for scanner.Scan() { + line := scanner.Text() + + // Track timestamp from console msgs + if tsMatch := timestampRegex.FindStringSubmatch(line); len(tsMatch) > 1 { + lastTimestamp = tsMatch[1] + } + + if match := toNetworkRegex.FindStringSubmatch(line); len(match) > 1 { + printAppInteraction("TO NETWORK", match[1], lastTimestamp, "") + } else if match := toNativeRegex.FindStringSubmatch(line); len(match) > 1 { + payload := cleanPayload(match[1]) + id := extractID(payload) + printAppInteraction("TO NATIVE", payload, lastTimestamp, id) + } else if match := fromNativeRegex.FindStringSubmatch(line); len(match) > 1 { + payload := cleanPayload(match[1]) + id := extractID(payload) + printAppInteraction("FROM NATIVE", payload, lastTimestamp, id) + } + } + + if err := scanner.Err(); err != nil { + fmt.Printf("Error reading file: %v\n", err) + } +} + +func cleanPayload(p string) string { + // Remove escaped quotes and leading/trailing quotes + p = strings.ReplaceAll(p, `\"`, `"`) + return p +} + +func extractID(p string) string { + // Try to find "id":X + idRegex := regexp.MustCompile(`"id":\s*(\d+)`) + match := idRegex.FindStringSubmatch(p) + if len(match) > 1 { + return match[1] + } + return "" +} + +func printAppInteraction(typ, payload, ts, id string) { + fmt.Printf("\n### %s", typ) + if ts != "" { + fmt.Printf(" [%s]", ts) + } + if id != "" { + fmt.Printf(" (ID: %s)", id) + } + fmt.Println() + + // Try to prettify if it's JSON + var obj interface{} + if err := json.Unmarshal([]byte(payload), &obj); err == nil { + pretty, _ := json.MarshalIndent(obj, "", " ") + fmt.Printf("/*\n%s\n*/\n", string(pretty)) + } else { + // Just print raw (might be XML or plain text) + fmt.Printf("/*\n%s\n*/\n", payload) + } + fmt.Println("-------------------------------------------------") +} diff --git a/scripts/extract-ws/main.go b/scripts/extract-ws/main.go new file mode 100644 index 0000000..39797a2 --- /dev/null +++ b/scripts/extract-ws/main.go @@ -0,0 +1,254 @@ +package main + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/google/gopacket/pcap" +) + +// This tool extracts WebSocket payloads and DNS queries from a .pcap file +// and prints them in a format compatible with soundtouch-service interactions. + +func main() { + if len(os.Args) < 2 { + fmt.Println("Usage: go run scripts/extract-ws.go [filter_ip]") + os.Exit(1) + } + + pcapFile := os.Args[1] + filterIP := "" + if len(os.Args) > 2 { + filterIP = os.Args[2] + fmt.Printf("[DEBUG] Filtering WebSocket for IP: %s\n", filterIP) + } + + handle, err := pcap.OpenOffline(pcapFile) + if err != nil { + log.Fatal(err) + } + defer handle.Close() + + fmt.Printf("[DEBUG] Reading file: %s\n", pcapFile) + + // Prepare output files + baseName := strings.TrimSuffix(pcapFile, filepath.Ext(pcapFile)) + wsFile, err := os.Create(baseName + ".ws.http") + if err != nil { + log.Fatal(err) + } + defer wsFile.Close() + + dnsFile, err := os.Create(baseName + ".dns.txt") + if err != nil { + log.Fatal(err) + } + defer dnsFile.Close() + + mdnsFile, err := os.Create(baseName + ".mdns.txt") + if err != nil { + log.Fatal(err) + } + defer mdnsFile.Close() + + packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) + + for packet := range packetSource.Packets() { + // Handle DNS + if dnsLayer := packet.Layer(layers.LayerTypeDNS); dnsLayer != nil { + dns, _ := dnsLayer.(*layers.DNS) + extractDNS(packet, dns, dnsFile, mdnsFile) + } + + // Handle SSDP (UDP Port 1900) + if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil { + udp, _ := udpLayer.(*layers.UDP) + if udp.DstPort == 1900 || udp.SrcPort == 1900 { + extractSSDP(packet, udp, baseName+".ssdp.txt") + } + } + + // Handle WebSockets (TCP) + if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil { + tcp, _ := tcpLayer.(*layers.TCP) + extractWebSocket(packet, tcp, filterIP, wsFile) + } + } + + fmt.Printf("[DEBUG] Extraction complete. Results written to:\n- %s\n- %s\n- %s\n- %s\n", + baseName+".ws.http", baseName+".dns.txt", baseName+".mdns.txt", baseName+".ssdp.txt") +} + +func extractSSDP(packet gopacket.Packet, udp *layers.UDP, ssdpFilename string) { + payload := string(udp.Payload) + if !strings.Contains(payload, "HTTP/1.1") && !strings.Contains(payload, "NOTIFY") && !strings.Contains(payload, "M-SEARCH") { + return + } + + f, err := os.OpenFile(ssdpFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return + } + defer f.Close() + + srcIP := packet.NetworkLayer().NetworkFlow().Src().String() + dstIP := packet.NetworkLayer().NetworkFlow().Dst().String() + timestamp := packet.Metadata().Timestamp.Format("2006-01-02 15:04:05.000") + + fmt.Fprintf(f, "[%s] %s:%d -> %s:%d\n", timestamp, srcIP, udp.SrcPort, dstIP, udp.DstPort) + fmt.Fprintf(f, "%s\n", strings.TrimSpace(payload)) + fmt.Fprintln(f, "-------------------------------------------------") +} + +func extractDNS(packet gopacket.Packet, dns *layers.DNS, dnsFile, mdnsFile *os.File) { + srcIP := packet.NetworkLayer().NetworkFlow().Src().String() + dstIP := packet.NetworkLayer().NetworkFlow().Dst().String() + + isMDNS := false + if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil { + udp, _ := udpLayer.(*layers.UDP) + if udp.DstPort == 5353 || udp.SrcPort == 5353 { + isMDNS = true + } + } + + out := dnsFile + if isMDNS { + out = mdnsFile + } + + timestamp := packet.Metadata().Timestamp.Format("2006-01-02 15:04:05.000") + prefix := fmt.Sprintf("[%s] %s -> %s", timestamp, srcIP, dstIP) + + for _, q := range dns.Questions { + fmt.Fprintf(out, "%s | QUERY: %s (%s)\n", prefix, string(q.Name), q.Type) + } + for _, a := range dns.Answers { + val := "" + if a.IP != nil { + val = a.IP.String() + } else if len(a.CNAME) > 0 { + val = string(a.CNAME) + } else if len(a.PTR) > 0 { + val = string(a.PTR) + } else if len(a.TXTs) > 0 { + var txts []string + for _, t := range a.TXTs { + txts = append(txts, string(t)) + } + val = strings.Join(txts, " ") + } else { + val = fmt.Sprintf("Type: %s", a.Type) + } + fmt.Fprintf(out, "%s | ANSWER: %s -> %s\n", prefix, string(a.Name), val) + } +} + +func extractWebSocket(packet gopacket.Packet, tcp *layers.TCP, filterIP string, wsFile *os.File) { + srcIP := packet.NetworkLayer().NetworkFlow().Src().String() + dstIP := packet.NetworkLayer().NetworkFlow().Dst().String() + + if filterIP != "" && srcIP != filterIP && dstIP != filterIP { + return + } + + payload := tcp.Payload + if len(payload) == 0 { + return + } + + // Check for WebSocket Frame (Sliding search) + for i := 0; i < len(payload)-2; i++ { + firstByte := payload[i] + // Opcode 1 (Text) or 2 (Binary). + if (firstByte&0xF0) == 0x80 && (firstByte&0x0F == 1 || firstByte&0x0F == 2) { + secondByte := payload[i+1] + mask := (secondByte & 0x80) != 0 + length := int(secondByte & 0x7F) + offset := i + 2 + + if length == 126 { + if len(payload) < offset+2 { + continue + } + length = int(payload[offset])<<8 | int(payload[offset+1]) + offset += 2 + } else if length == 127 { + if len(payload) < offset+8 { + continue + } + length = int(payload[offset+4])<<24 | int(payload[offset+5])<<16 | int(payload[offset+6])<<8 | int(payload[offset+7]) + offset += 8 + } + + if mask { + if len(payload) < offset+4+length { + continue + } + maskKey := payload[offset : offset+4] + offset += 4 + data := make([]byte, length) + for j := 0; j < length; j++ { + data[j] = payload[offset+j] ^ maskKey[j%4] + } + printInteraction(packet, tcp, data, wsFile) + i = offset + length - 1 + } else { + if len(payload) >= offset+length { + data := payload[offset : offset+length] + printInteraction(packet, tcp, data, wsFile) + i = offset + length - 1 + } + } + } + } +} + +func printInteraction(packet gopacket.Packet, tcp *layers.TCP, data []byte, out io.Writer) { + src := packet.NetworkLayer().NetworkFlow().Src().String() + dst := packet.NetworkLayer().NetworkFlow().Dst().String() + + fmt.Fprintf(out, "### WebSocket Message: %s -> %s\n", src, dst) + fmt.Fprintf(out, "// Timestamp: %s\n", packet.Metadata().Timestamp) + fmt.Fprintf(out, "// Ports: %d -> %d\n", tcp.SrcPort, tcp.DstPort) + fmt.Fprintln(out) + + // Try to detect if it's GZIP + content := "" + if len(data) > 2 && data[0] == 0x1f && data[1] == 0x8b { + fmt.Fprintln(out, "// [Detected GZIP compression]") + content = decompressGzip(data) + } else { + content = string(data) + } + + fmt.Fprintln(out, "/*") + fmt.Fprintln(out, strings.TrimSpace(content)) + fmt.Fprintln(out, "*/") + fmt.Fprintln(out, "") + fmt.Fprintln(out, "-------------------------------------------------") + fmt.Fprintln(out, "") +} + +func decompressGzip(data []byte) string { + b := bytes.NewBuffer(data) + r, err := gzip.NewReader(b) + if err != nil { + return "[Error: Failed to create GZIP reader: " + err.Error() + "]" + } + defer r.Close() + + res, err := io.ReadAll(r) + if err != nil { + return "[Error: Failed to decompress GZIP: " + err.Error() + "]" + } + return string(res) +} diff --git a/scripts/http-diff/README.md b/scripts/http-diff/README.md new file mode 100644 index 0000000..98d8625 --- /dev/null +++ b/scripts/http-diff/README.md @@ -0,0 +1,32 @@ +# HTTP Body Diff Tool + +This tool extracts and compares response bodies from two `.http` files. +It supports XML and JSON normalization (pretty-printing) and automatically masks common "noisy" fields like timestamps to make actual differences easier to spot. + +## Usage + +```bash +go run scripts/http-diff/main.go +``` + +To generate a side-by-side HTML report: + +```bash +go run scripts/http-diff/main.go --html report.html +``` + +## Features + +- **Body Extraction**: Automatically finds the response body within the `/* ... */` comment block at the end of the file. +- **Side-by-Side View**: Generates an HTML report with a clear side-by-side comparison. +- **Normalization**: + - Pretty-prints XML and JSON. + - Trims whitespace from XML character data. +- **Noise Reduction**: + - Automatically replaces ISO 8601 timestamps with `[TIMESTAMP]`. + - Masks specific XML tags: ``, ``, ``, ``. + - Masks specific JSON keys: `timestamp`, `updatedOn`, `createdOn`, `expires_at`. +- **Diff Output**: + - Displays a line-by-line diff. + - Show context for unchanged parts (first and last two lines, with `...` in between). + - Uses `+` for additions and `-` for deletions. diff --git a/scripts/http-diff/main.go b/scripts/http-diff/main.go new file mode 100644 index 0000000..587d9fb --- /dev/null +++ b/scripts/http-diff/main.go @@ -0,0 +1,299 @@ +package main + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "flag" + "fmt" + "html" + "io" + "os" + "regexp" + "strings" + + "github.com/sergi/go-diff/diffmatchpatch" +) + +func main() { + htmlOutput := flag.String("html", "", "Path to save HTML diff report") + flag.Parse() + + args := flag.Args() + if len(args) < 2 { + fmt.Println("Usage: http-diff [options] ") + fmt.Println("Options:") + flag.PrintDefaults() + os.Exit(1) + } + + file1 := args[0] + file2 := args[1] + + body1, err := extractBody(file1) + if err != nil { + fmt.Printf("Error extracting body from %s: %v\n", file1, err) + os.Exit(1) + } + + body2, err := extractBody(file2) + if err != nil { + fmt.Printf("Error extracting body from %s: %v\n", file2, err) + os.Exit(1) + } + + norm1 := normalize(body1) + norm2 := normalize(body2) + + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(norm1, norm2, false) + lineDiffs := dmp.DiffCleanupSemantic(diffs) + + if *htmlOutput != "" { + err := generateHTML(*htmlOutput, file1, file2, lineDiffs) + if err != nil { + fmt.Printf("Error generating HTML: %v\n", err) + os.Exit(1) + } + fmt.Printf("HTML report generated: %s\n", *htmlOutput) + return + } + + // Custom line-by-line diff for better readability + for _, diff := range lineDiffs { + switch diff.Type { + case diffmatchpatch.DiffInsert: + lines := strings.Split(diff.Text, "\n") + for _, line := range lines { + if line != "" { + fmt.Printf("+ %s\n", line) + } + } + case diffmatchpatch.DiffDelete: + lines := strings.Split(diff.Text, "\n") + for _, line := range lines { + if line != "" { + fmt.Printf("- %s\n", line) + } + } + case diffmatchpatch.DiffEqual: + // Optionally skip unchanged lines or show context + lines := strings.Split(diff.Text, "\n") + // Filter out empty lines from splitting + var cleanLines []string + for _, l := range lines { + if strings.TrimSpace(l) != "" { + cleanLines = append(cleanLines, l) + } + } + + if len(cleanLines) > 6 { + fmt.Printf(" %s\n", cleanLines[0]) + fmt.Printf(" %s\n", cleanLines[1]) + fmt.Printf(" ...\n") + fmt.Printf(" %s\n", cleanLines[len(cleanLines)-2]) + fmt.Printf(" %s\n", cleanLines[len(cleanLines)-1]) + } else { + for _, line := range cleanLines { + fmt.Printf(" %s\n", line) + } + } + } + } +} + +func extractBody(path string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + + // Use a non-greedy regex to find the LAST /* ... */ block which typically contains the body + re := regexp.MustCompile(`(?s)/\*\s*(<\?xml.*?|\{.*?|\[.*?)\s*\*/`) + matches := re.FindAllStringSubmatch(string(content), -1) + if len(matches) > 0 { + // Return the last match as it's more likely to be the response body + lastMatch := matches[len(matches)-1] + return strings.TrimSpace(lastMatch[1]), nil + } + + return "", fmt.Errorf("could not find body in /* ... */ block") +} + +func normalize(body string) string { + body = strings.TrimSpace(body) + if body == "" { + return "" + } + + // Mask common timestamps and changing fields + timestampRegex := regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})`) + body = timestampRegex.ReplaceAllString(body, "[TIMESTAMP]") + + // Mask account IDs if they are variable, but usually they match in these files. + // Let's stick to timestamps for now. + + // Try XML first + if strings.HasPrefix(body, " or + tagsToMask := []string{"updatedOn", "createdOn", "lastModified", "timestamp"} + for _, tag := range tagsToMask { + re := regexp.MustCompile(fmt.Sprintf(`<%s>.*?`, tag, tag)) + body = re.ReplaceAllString(body, fmt.Sprintf("<%s>[MASKED]", tag, tag)) + } + + // Also mask empty attributes that might be noisy, like displayName="" + body = regexp.MustCompile(`\s+displayName=""`).ReplaceAllString(body, "") + + var out bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(body)) + encoder := xml.NewEncoder(&out) + encoder.Indent("", " ") + for { + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + // If it's not valid XML, maybe it's just a fragment, continue or return body + break + } + // Trim whitespace from CharData to normalize + if cd, ok := token.(xml.CharData); ok { + token = xml.CharData(bytes.TrimSpace(cd)) + } + err = encoder.EncodeToken(token) + if err != nil { + break + } + } + encoder.Flush() + if out.Len() > 0 { + return out.String() + } + } + + // Try JSON + if strings.HasPrefix(body, "{") || strings.HasPrefix(body, "[") { + var obj interface{} + if err := json.Unmarshal([]byte(body), &obj); err == nil { + // Mask some JSON fields if they are common + maskJSON(obj) + pretty, _ := json.MarshalIndent(obj, "", " ") + return string(pretty) + } + } + + return body +} + +func maskJSON(data interface{}) { + switch v := data.(type) { + case map[string]interface{}: + for k, val := range v { + if strings.Contains(strings.ToLower(k), "timestamp") || k == "updatedOn" || k == "createdOn" || k == "expires_at" { + v[k] = "[MASKED]" + } else { + maskJSON(val) + } + } + case []interface{}: + for _, item := range v { + maskJSON(item) + } + } +} + +func generateHTML(path, file1, file2 string, diffs []diffmatchpatch.Diff) error { + var sb strings.Builder + sb.WriteString(` + + + + HTTP Diff Report + + + +
+
+

HTTP Response Body Diff

+
+ Left: ` + html.EscapeString(file1) + `
+ Right: ` + html.EscapeString(file2) + ` +
+
+ +`) + + type sideBySideLine struct { + leftText string + rightText string + class string + } + var lines []sideBySideLine + + for _, diff := range diffs { + text := html.EscapeString(diff.Text) + split := strings.Split(text, "\n") + // Remove trailing empty string from split if it exists + if len(split) > 0 && split[len(split)-1] == "" { + split = split[:len(split)-1] + } + + switch diff.Type { + case diffmatchpatch.DiffEqual: + for _, line := range split { + lines = append(lines, sideBySideLine{leftText: line, rightText: line, class: "diff-equal"}) + } + case diffmatchpatch.DiffInsert: + for _, line := range split { + lines = append(lines, sideBySideLine{leftText: "", rightText: line, class: "diff-insert"}) + } + case diffmatchpatch.DiffDelete: + for _, line := range split { + lines = append(lines, sideBySideLine{leftText: line, rightText: "", class: "diff-delete"}) + } + } + } + + for i, line := range lines { + leftMarker := "" + rightMarker := "" + if line.class == "diff-insert" { + rightMarker = "+" + } else if line.class == "diff-delete" { + leftMarker = "-" + } + + sb.WriteString(fmt.Sprintf(` + + + + + + `, line.class, i+1, leftMarker, line.leftText, i+1, rightMarker, line.rightText)) + } + + sb.WriteString(` +
%d
%s%s
%d
%s%s
+
+ + +`) + + return os.WriteFile(path, []byte(sb.String()), 0644) +}