Add app analyzing/debugging docs and scripts (#174)

This commit is contained in:
Tobias Gesellchen
2026-04-19 22:27:54 +02:00
committed by GitHub
parent 88c83b6131
commit 747a9cec97
12 changed files with 1223 additions and 23 deletions
+9
View File
@@ -34,6 +34,9 @@ jobs:
restore-keys: |
${{ runner.os }}-go-
- name: Install libpcap
run: sudo apt-get install -y libpcap-dev
- name: Download dependencies
run: go mod download
@@ -70,6 +73,9 @@ jobs:
with:
go-version-file: "go.mod"
- name: Install libpcap
run: sudo apt-get install -y libpcap-dev
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v9
with:
@@ -127,6 +133,9 @@ jobs:
with:
go-version-file: "go.mod"
- name: Install libpcap
run: sudo apt-get install -y libpcap-dev
- name: Run basic vulnerability check
run: |
go install golang.org/x/vuln/cmd/govulncheck@latest
+3
View File
@@ -68,6 +68,9 @@ jobs:
with:
go-version-file: ${{ env.GO_VERSION_FILE }}
- name: Install libpcap
run: sudo apt-get install -y libpcap-dev
- name: Run tests before release
run: |
echo "Running final tests before release..."
+9
View File
@@ -26,6 +26,9 @@ jobs:
with:
go-version-file: "go.mod"
- name: Install libpcap
run: sudo apt-get install -y libpcap-dev
- name: Install security scanning tools
run: |
go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -67,6 +70,9 @@ jobs:
with:
go-version-file: "go.mod"
- name: Install libpcap
run: sudo apt-get install -y libpcap-dev
- name: Install static analysis tools
run: |
go install honnef.co/go/tools/cmd/staticcheck@latest
@@ -106,6 +112,9 @@ jobs:
- name: Checkout code
uses: actions/checkout@v6
- name: Install libpcap
run: sudo apt-get install -y libpcap-dev
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
with:
+1
View File
@@ -60,6 +60,7 @@
* [IoT Configuration Analysis](analysis/IOT-CONFIGURATION-ANALYSIS.md)
* [Bose Lab Runbook](analysis/BOSE-LAB-RUNBOOK.md)
* [Missing Routes Spotify](analysis/MISSING-ROUTES-SPOTIFY.md)
* [Bose App ADB Emulator](analysis/BOSE-APP-ADB-Emulator.md)
## Parity Analysis
* [Parity Improvements](PARITY-IMPROVEMENTS.md)
+341
View File
@@ -0,0 +1,341 @@
# Bose SoundTouch Traffic Interception Runbook
Intercept HTTPS/WebSocket traffic from the Bose SoundTouch Android app using an Android emulator, mitmproxy, and Frida. Tested on Apple Silicon (ARM64) Mac.
## Prerequisites
- Android Studio installed (for SDK tools and emulator)
- Docker installed
- mitmproxy installed (`pip install mitmproxy` or via your preferred method)
- The Bose SoundTouch APK (extracted from a real device, see below)
Add Android SDK tools to your PATH (add to `~/.zshrc`):
```bash
export PATH=$PATH:~/Library/Android/sdk/emulator
export PATH=$PATH:~/Library/Android/sdk/platform-tools
```
---
## 1. Extract APK from Real Device
Connect your Android device via USB with USB debugging enabled.
```bash
adb devices
# note your device ID, e.g. "ABC123"
adb -s ABC123 shell pm path com.bose.soundtouch
# output e.g.: package:/data/app/~~xyz/com.bose.soundtouch-abc/base.apk
adb -s ABC123 pull /data/app/~~xyz/com.bose.soundtouch-abc/base.apk bose.apk
```
---
## 2. Create Android Emulator (ARM64, API 33)
On Apple Silicon you need an ARM64 image. Use the `avdmanager` and `sdkmanager` CLI tools.
```bash
# Install the system image
~/Library/Android/sdk/cmdline-tools/latest/bin/sdkmanager \
"system-images;android-33;google_apis;arm64-v8a"
# Create the AVD
~/Library/Android/sdk/cmdline-tools/latest/bin/avdmanager create avd \
-n Pixel_6_API33 \
-k "system-images;android-33;google_apis;arm64-v8a" \
-d "pixel_6"
```
Alternatively create the AVD via Android Studio Device Manager (choose "Google APIs", arm64-v8a, API 33).
---
## 3. Start Emulator with Writable System
```bash
# List available AVDs
~/Library/Android/sdk/emulator/emulator -list-avds
# Start with writable system partition
~/Library/Android/sdk/emulator/emulator -avd Pixel_6_API33 -writable-system
```
Wait until the emulator has fully booted, then:
```bash
adb -s emulator-5554 root
adb -s emulator-5554 shell avbctl disable-verification
adb -s emulator-5554 reboot
# After reboot:
adb -s emulator-5554 root
```
---
## 4. Install Bose APK
```bash
adb -s emulator-5554 install bose.apk
```
---
## 5. Set Up mitmproxy
```bash
# Start mitmproxy (generates CA cert on first run)
mitmweb --port 8080 --mode regular -w bose_traffic.mitm
```
Extract the CA certificate (without private key):
```bash
openssl x509 -in ~/.mitmproxy/mitmproxy-ca.pem -out ~/.mitmproxy/mitmproxy-ca-cert.pem
# Verify it's the mitmproxy cert, not another cert:
openssl x509 -in ~/.mitmproxy/mitmproxy-ca-cert.pem -noout -issuer
# should show: issuer= /CN=mitmproxy/O=mitmproxy
```
---
## 6. Install mitmproxy CA Certificate in Emulator
```bash
HASH=$(openssl x509 -inform PEM -subject_hash_old \
-in ~/.mitmproxy/mitmproxy-ca-cert.pem | head -1)
adb -s emulator-5554 push ~/.mitmproxy/mitmproxy-ca-cert.pem /data/local/tmp/mitmproxy.pem
adb -s emulator-5554 shell su 0 mkdir -p /data/misc/user/0/cacerts-added
adb -s emulator-5554 shell su 0 \
cp /data/local/tmp/mitmproxy.pem /data/misc/user/0/cacerts-added/${HASH}.0
adb -s emulator-5554 shell su 0 \
chmod 644 /data/misc/user/0/cacerts-added/${HASH}.0
```
---
## 7. Set System Proxy in Emulator
Find your Mac's local IP:
```bash
ipconfig getifaddr en0
# e.g. 192.168.1.123
```
Set the proxy:
```bash
adb -s emulator-5554 shell settings put global http_proxy 192.168.1.123:8080
```
---
## 8. Set Up Frida (via Python venv)
```bash
python3 -m venv /tmp/frida-venv
/tmp/frida-venv/bin/pip install frida==17.9.1 frida-tools==14.8.1
```
Download the frida-server binary for ARM64 Android:
```bash
FRIDA_VERSION=17.9.1
curl -L "https://github.com/frida/frida/releases/download/${FRIDA_VERSION}/frida-server-${FRIDA_VERSION}-android-arm64.xz" \
-o /tmp/frida-server.xz
unxz /tmp/frida-server.xz
mv /tmp/frida-server-${FRIDA_VERSION}-android-arm64 /tmp/frida-server
```
Push to emulator and start:
```bash
adb -s emulator-5554 push /tmp/frida-server /data/local/tmp/frida-server
adb -s emulator-5554 shell su 0 chmod 755 /data/local/tmp/frida-server
adb -s emulator-5554 shell su 0 /data/local/tmp/frida-server &
```
---
## 9. Download SSL Bypass Scripts
```bash
BASE=https://raw.githubusercontent.com/httptoolkit/frida-interception-and-unpinning/main
curl -L "${BASE}/config.js" -o /tmp/config.js
curl -L "${BASE}/android/android-system-certificate-injection.js" \
-o /tmp/android-system-certificate-injection.js
curl -L "${BASE}/android/android-proxy-override.js" \
-o /tmp/android-proxy-override.js
curl -L "${BASE}/android/android-certificate-unpinning.js" \
-o /tmp/android-certificate-unpinning.js
curl -L "${BASE}/android/android-certificate-unpinning-fallback.js" \
-o /tmp/android-certificate-unpinning-fallback.js
```
---
## 10. Configure config.js
Edit `/tmp/config.js` and set:
```javascript
const CERT_PEM = `<contents of ~/.mitmproxy/mitmproxy-ca-cert.pem>`;
const PROXY_HOST = '192.168.1.123'; // your Mac IP
const PROXY_PORT = 8080;
```
Insert the full PEM content (from `-----BEGIN CERTIFICATE-----` to `-----END CERTIFICATE-----`) between the backticks.
Quick check that the right cert is in place:
```bash
# The issuer inside config.js should be mitmproxy, not SoundTouch
grep -A3 "CERT_PEM" /tmp/config.js | head -5
```
---
## 11. Start Interception
Make sure mitmweb is running, then:
```bash
/tmp/frida-venv/bin/frida \
-U \
-f com.bose.soundtouch \
-l /tmp/config.js \
-l /tmp/android-system-certificate-injection.js \
-l /tmp/android-proxy-override.js \
-l /tmp/android-certificate-unpinning.js \
-l /tmp/android-certificate-unpinning-fallback.js
```
Expected output in the Frida REPL:
```
== System certificate trust injected ==
== Proxy system configuration overridden to 192.168.1.123:8080 ==
== Proxy configuration overridden to 192.168.1.123:8080 ==
== Certificate unpinning completed ==
== Unpinning fallback auto-patcher installed ==
```
Open mitmweb at `http://127.0.0.1:8081` to observe traffic live.
---
## 12. Save & Replay Recordings
Traffic is saved to `bose_traffic.mitm` (set via `-w` flag in step 5).
```bash
# Replay/analyse a saved recording:
mitmweb -r bose_traffic.mitm
```
---
## Cleanup
```bash
# Remove proxy setting from emulator
adb -s emulator-5554 shell settings delete global http_proxy
# Remove venv
rm -rf /tmp/frida-venv /tmp/frida-server /tmp/frida-server.xz
rm /tmp/config.js /tmp/android-*.js
# Stop emulator
adb -s emulator-5554 emu kill
```
---
## Troubleshooting
| Symptom | Cause | Fix |
|-----------------------------------------|--------------------------------------------------|--------------------------------------------------------------------------------|
| `remount failed` | ARM64 emulator doesn't support overlayfs remount | Use `/data/misc/user/0/cacerts-added/` method instead |
| `TLS: Trust anchor not found` | Wrong certificate in config.js | Check issuer: must be mitmproxy, not SoundTouch |
| `Chain validation failed` | Private key included in cert | Re-extract with `openssl x509 -in mitmproxy-ca.pem -out mitmproxy-ca-cert.pem` |
| `frida-server: connection refused` | frida-server not running | Re-run `adb shell su 0 /data/local/tmp/frida-server &` |
| frida and frida-server version mismatch | Versions must be identical | Pin both to same version (e.g. `17.9.1`) |
| `emulator: multiple AVDs` error | Emulator already running | Kill first: `adb emu kill`, then restart with `-writable-system` |
---
## App Automation Options
For most traffic-recording purposes, manually operating the app while mitmproxy captures is sufficient. If you need to automate specific interactions (e.g. to repeatably capture the requests triggered by startup or a particular action), the following tools are available.
### Starting the App
```bash
# Via app drawer: swipe up on the home screen and tap "Bose SoundTouch"
# Via adb monkey (simplest)
adb -s emulator-5554 shell monkey -p com.bose.soundtouch 1
# Via explicit intent (if the activity name is known)
adb -s emulator-5554 shell am start -n com.bose.soundtouch/.MainActivity
# Look up all activities if the name is unknown
adb -s emulator-5554 shell dumpsys package com.bose.soundtouch | grep Activity
```
### adb — sufficient for simple cases
```bash
# Tap at screen coordinates
adb shell input tap 540 960
# Swipe
adb shell input swipe 540 1500 540 500
# Type text
adb shell input text "mytext"
# Take a screenshot
adb shell screencap /sdcard/screen.png && adb pull /sdcard/screen.png
```
### UIAutomator2 — inspect UI elements
```bash
# Dump the current UI hierarchy to find element IDs
adb shell uiautomator dump /sdcard/ui.xml
adb pull /sdcard/ui.xml
```
Open `ui.xml` to find element resource IDs, then target them precisely in scripts.
### Appium — full scripted automation
```python
from appium import webdriver
driver = webdriver.Remote('http://localhost:4723/wd/hub', {
'platformName': 'Android',
'appPackage': 'com.bose.soundtouch',
'appActivity': '.MainActivity',
})
# Find an element by resource ID and tap it
driver.find_element('id', 'com.bose.soundtouch:id/play_button').click()
```
> **Note:** `monkey` is a stress-test tool that sends random events — use it only to launch the app, not to drive specific interactions.
+138 -23
View File
@@ -585,7 +585,72 @@ adb install Bose-SoundTouch-patched.apk
2. On the phone, use a File Manager to open the APK.
3. If prompted, allow "Install from Unknown Sources" for your File Manager.
### Option C: Patching the App with Frida (Requires Root)
### Option C: Using the macOS Bose SoundTouch App (No Root/Patching Required)
If you have a Mac, using the macOS version of the Bose SoundTouch app is often a good alternative. However, because the app is built on an **older version of Qt (5.7.0)**, it has specific trust and TLS compatibility issues that require extra steps.
#### 1. Install the Custom CA in macOS Keychain
1. Open **Keychain Access** on your Mac.
2. Select the **System** keychain (or **login** if System is locked).
3. Drag and drop your `ca.crt` file into the list.
4. Double-click the newly added certificate (e.g., "Bose-Lab Root CA").
5. Expand the **Trust** section.
6. Set "When using this certificate" to **Always Trust**.
7. Close the window and authenticate with your Mac password.
#### 2. Configure the Proxy
You can either configure the macOS system proxy manually or use `mitmproxy`'s automatic interception.
**Method 1: System Proxy (Manual)**
1. Go to **System Settings → Network → Wi-Fi → Details... → Proxies**.
2. Enable **HTTP Proxy** and **HTTPS Proxy**.
3. Set Server to your Pi's IP (`192.168.10.1`) and Port to `8080`.
4. Click **OK** and **Apply**.
**Method 2: mitmproxy Local Redirect (Automatic)**
If you are running `mitmproxy` directly on your Mac (instead of the Pi), you can use the modern "Local Redirect" mode which doesn't require proxy settings:
```bash
# Install mitmproxy via Homebrew
brew install mitmproxy
# Start mitmproxy in local redirect mode
# This uses a macOS Network Extension to intercept traffic from specific apps
mitmproxy --mode local
```
#### 3. Special Troubleshooting: Legacy Qt 5.7.0 SSL Failures
If you see `SSL handshake failed` in the `mitmproxy` logs or the app's internal log (`log.txt`), the app's older networking stack is rejecting the connection. This is common because Qt 5.7.0 (2016) lacks support for **TLS 1.3** and many modern root certificates (like Let's Encrypt's **ISRG Root X1**).
**The Solution: Launch with SSL Bypass Flags**
Since the Bose macOS app is a hybrid of **Qt/Chromium** and **Node.js**, you must bypass the trust checks for both engines by launching the app from the terminal:
```bash
# 1. Bypass QtWebEngine/Chromium (Qt 5.7) trust
export QTWEBENGINE_CHROMIUM_FLAGS="--ignore-certificate-errors"
# 2. Bypass Node.js (SoundTouch Music Server) trust
export NODE_TLS_REJECT_UNAUTHORIZED=0
# 3. (Optional) Provide your custom CA directly to Node.js
export NODE_EXTRA_CA_CERTS="/path/to/your/ca.crt"
# 4. Launch the application
"/Applications/SoundTouch/SoundTouch.app/Contents/MacOS/SoundTouch"
```
#### 4. Verify and Capture
1. Open Safari and visit `https://neverssl.com`. Verify the certificate is issued by your custom CA.
2. Launch the Bose app using the terminal command above.
3. Watch the traffic flow in `mitmproxy`.
> **Note:** Even on macOS, **Certificate Pinning** is still possible if Bose implemented it specifically in the desktop app code. However, it is much less common on desktop apps than on mobile apps. If it works, you've saved yourself hours of Android patching!
### Option D: Patching the App with Frida (Requires Root)
If the app uses **Certificate Pinning** (hardcoded hashes), even moving the CA to the System store won't work. You must disable the pinning check in the app's code.
@@ -625,38 +690,57 @@ mitmproxy --listen-port 8080
## Step 13 Extracting for soundtouch-service
You can extract interactions (especially unencrypted WebSockets on port 8090) from a `.pcap` and format them for use in `soundtouch-service`.
### 1. Extract Traffic using Go
A helper script is provided in `scripts/extract-ws.go`. It automatically detects, unmasks, and decompresses (GZIP) WebSocket frames, and also extracts DNS, MDNS, and SSDP traffic.
```bash
# Which IPs did the phone receive?
cat /var/lib/misc/dnsmasq.leases
# Install dependencies
go get github.com/google/gopacket
# Is the access point active?
sudo systemctl status hostapd
# Run extraction (outputs multiple files: .ws.http, .dns.txt, .mdns.txt, .ssdp.txt)
# The results will be saved beside your .pcap file
go run scripts/extract-ws.go your_capture.pcap [filter_ip]
# Is dnsmasq active?
sudo systemctl status dnsmasq
# Example: Filter for a specific speaker's IP in WebSocket messages
go run scripts/extract-ws.go capture.pcap 192.168.100.1
```
# Check interfaces and IPs
ip addr show
### 2. Manual Extraction with tshark
# Check routing table
ip route show
If you only need a quick look at the payloads:
# Show active nftables rules
sudo nft list ruleset
# All running tcpdump processes
pgrep -a tcpdump
# Test the Pi's own DNS resolution
dig @127.0.0.1 -p 5353 global.api.bose.io
# Check network connectivity from the phone (from the Pi)
ping 192.168.10.101 # Phone IP from dnsmasq.leases
```bash
# Extract all WebSocket text payloads
tshark -r your_capture.pcap -Y "websocket.payload.text" -T fields -e websocket.payload.text
```
---
## Restart Sequence
## Step 14 Extracting from Internal App Logs (macOS)
If you are using the macOS app and cannot decrypt the cloud traffic due to pinning, you can still extract the JSON/XML messages from the app's internal communication log.
A helper script is provided in `scripts/extract-log-interactions.go`. It parses the interleaved "Native" and "Network" calls to reconstruct the application's internal state and cloud requests.
```bash
# Run extraction from the log file
# Outputs a chronological record of internal events and network URLs
go run scripts/extract-log-interactions.go path/to/log.txt > extracted-interactions.http
```
**What this shows:**
- **TO NETWORK:** The URLs the app is about to call (intercepted before encryption).
- **FROM NATIVE:** Data being returned from the OS or Cloud to the UI.
- **TO NATIVE:** Commands being sent from the UI to the underlying engines.
This is a powerful "Plan B" when HTTPS decryption is blocked, as the app essentially logs its own decrypted data for you.
---
## Helper Commands / Troubleshooting
After a Pi reboot, everything should come up automatically. If not:
@@ -775,3 +859,34 @@ sudo openssl x509 -req -in bose.csr -CA ca.crt -CAkey ca.key \
### 3. Usage in your DNS/HTTPS Server
Your custom server (e.g., a small Go or Python script) would then use `bose.crt` and `bose.key` to serve HTTPS traffic for those domains.
## Appendix B Helpful Commands
```bash
# Which IPs did the phone receive?
cat /var/lib/misc/dnsmasq.leases
# Is the access point active?
sudo systemctl status hostapd
# Is dnsmasq active?
sudo systemctl status dnsmasq
# Check interfaces and IPs
ip addr show
# Check routing table
ip route show
# Show active nftables rules
sudo nft list ruleset
# All running tcpdump processes
pgrep -a tcpdump
# Test the Pi's own DNS resolution
dig @127.0.0.1 -p 5353 global.api.bose.io
# Check network connectivity from the phone (from the Pi)
ping 192.168.10.101 # Phone IP from dnsmasq.leases
```
+2
View File
@@ -4,10 +4,12 @@ go 1.26.2
require (
github.com/go-chi/chi/v5 v5.2.5
github.com/google/gopacket v1.1.19
github.com/gorilla/websocket v1.5.3
github.com/hashicorp/mdns v1.0.6
github.com/miekg/dns v1.1.72
github.com/russross/blackfriday/v2 v2.1.0
github.com/sergi/go-diff v1.4.0
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef
github.com/urfave/cli/v2 v2.27.7
+27
View File
@@ -1,28 +1,44 @@
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/mdns v1.0.6 h1:SV8UcjnQ/+C7KeJ/QeVD/mdN2EmzYfcGfufcuzxfCLQ=
github.com/hashicorp/mdns v1.0.6/go.mod h1:X4+yWh+upFECLOki1doUPaKpgNQII9gy4bUdCYKNhmM=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE=
github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q=
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ=
github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
@@ -32,6 +48,8 @@ golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
golang.org/x/image v0.39.0 h1:skVYidAEVKgn8lZ602XO75asgXBgLj9G/FE3RbuPFww=
golang.org/x/image v0.39.0/go.mod h1:sIbmppfU+xFLPIG0FoVUTvyBMmgng1/XAMhQ2ft0hpA=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -40,6 +58,7 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -62,6 +81,7 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -101,6 +121,7 @@ golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
@@ -109,3 +130,9 @@ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxb
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+108
View File
@@ -0,0 +1,108 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"os"
"regexp"
"strings"
)
type Interaction struct {
Type string `json:"type"` // ToNative, FromNative, ToNetwork
Payload string `json:"payload"` // Raw string
Parsed interface{} `json:"parsed"` // JSON if possible
Timestamp string `json:"timestamp"` // If available
ID string `json:"id"` // Internal ID if available
}
func main() {
if len(os.Args) < 2 {
fmt.Println("Usage: go run extract-log-interactions.go <log_file>")
return
}
logFile := os.Args[1]
file, err := os.Open(logFile)
if err != nil {
fmt.Printf("Error opening file: %v\n", err)
return
}
defer file.Close()
// Regex patterns
toNativeRegex := regexp.MustCompile(`To Native : "(.*)"`)
fromNativeRegex := regexp.MustCompile(`From Native : "(.*)"`)
toNetworkRegex := regexp.MustCompile(`To Network "(.*)"`)
timestampRegex := regexp.MustCompile(`Js_Console_Msg: "(\d{2}:\d{2}:\d{2}\.\d{3})`)
scanner := bufio.NewScanner(file)
lastTimestamp := ""
fmt.Println("### Bose SoundTouch Internal Log Interactions")
fmt.Println("-------------------------------------------------")
for scanner.Scan() {
line := scanner.Text()
// Track timestamp from console msgs
if tsMatch := timestampRegex.FindStringSubmatch(line); len(tsMatch) > 1 {
lastTimestamp = tsMatch[1]
}
if match := toNetworkRegex.FindStringSubmatch(line); len(match) > 1 {
printAppInteraction("TO NETWORK", match[1], lastTimestamp, "")
} else if match := toNativeRegex.FindStringSubmatch(line); len(match) > 1 {
payload := cleanPayload(match[1])
id := extractID(payload)
printAppInteraction("TO NATIVE", payload, lastTimestamp, id)
} else if match := fromNativeRegex.FindStringSubmatch(line); len(match) > 1 {
payload := cleanPayload(match[1])
id := extractID(payload)
printAppInteraction("FROM NATIVE", payload, lastTimestamp, id)
}
}
if err := scanner.Err(); err != nil {
fmt.Printf("Error reading file: %v\n", err)
}
}
func cleanPayload(p string) string {
// Remove escaped quotes and leading/trailing quotes
p = strings.ReplaceAll(p, `\"`, `"`)
return p
}
func extractID(p string) string {
// Try to find "id":X
idRegex := regexp.MustCompile(`"id":\s*(\d+)`)
match := idRegex.FindStringSubmatch(p)
if len(match) > 1 {
return match[1]
}
return ""
}
func printAppInteraction(typ, payload, ts, id string) {
fmt.Printf("\n### %s", typ)
if ts != "" {
fmt.Printf(" [%s]", ts)
}
if id != "" {
fmt.Printf(" (ID: %s)", id)
}
fmt.Println()
// Try to prettify if it's JSON
var obj interface{}
if err := json.Unmarshal([]byte(payload), &obj); err == nil {
pretty, _ := json.MarshalIndent(obj, "", " ")
fmt.Printf("/*\n%s\n*/\n", string(pretty))
} else {
// Just print raw (might be XML or plain text)
fmt.Printf("/*\n%s\n*/\n", payload)
}
fmt.Println("-------------------------------------------------")
}
+254
View File
@@ -0,0 +1,254 @@
package main
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"github.com/google/gopacket/pcap"
)
// This tool extracts WebSocket payloads and DNS queries from a .pcap file
// and prints them in a format compatible with soundtouch-service interactions.
func main() {
if len(os.Args) < 2 {
fmt.Println("Usage: go run scripts/extract-ws.go <pcap_file> [filter_ip]")
os.Exit(1)
}
pcapFile := os.Args[1]
filterIP := ""
if len(os.Args) > 2 {
filterIP = os.Args[2]
fmt.Printf("[DEBUG] Filtering WebSocket for IP: %s\n", filterIP)
}
handle, err := pcap.OpenOffline(pcapFile)
if err != nil {
log.Fatal(err)
}
defer handle.Close()
fmt.Printf("[DEBUG] Reading file: %s\n", pcapFile)
// Prepare output files
baseName := strings.TrimSuffix(pcapFile, filepath.Ext(pcapFile))
wsFile, err := os.Create(baseName + ".ws.http")
if err != nil {
log.Fatal(err)
}
defer wsFile.Close()
dnsFile, err := os.Create(baseName + ".dns.txt")
if err != nil {
log.Fatal(err)
}
defer dnsFile.Close()
mdnsFile, err := os.Create(baseName + ".mdns.txt")
if err != nil {
log.Fatal(err)
}
defer mdnsFile.Close()
packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
for packet := range packetSource.Packets() {
// Handle DNS
if dnsLayer := packet.Layer(layers.LayerTypeDNS); dnsLayer != nil {
dns, _ := dnsLayer.(*layers.DNS)
extractDNS(packet, dns, dnsFile, mdnsFile)
}
// Handle SSDP (UDP Port 1900)
if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil {
udp, _ := udpLayer.(*layers.UDP)
if udp.DstPort == 1900 || udp.SrcPort == 1900 {
extractSSDP(packet, udp, baseName+".ssdp.txt")
}
}
// Handle WebSockets (TCP)
if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
tcp, _ := tcpLayer.(*layers.TCP)
extractWebSocket(packet, tcp, filterIP, wsFile)
}
}
fmt.Printf("[DEBUG] Extraction complete. Results written to:\n- %s\n- %s\n- %s\n- %s\n",
baseName+".ws.http", baseName+".dns.txt", baseName+".mdns.txt", baseName+".ssdp.txt")
}
func extractSSDP(packet gopacket.Packet, udp *layers.UDP, ssdpFilename string) {
payload := string(udp.Payload)
if !strings.Contains(payload, "HTTP/1.1") && !strings.Contains(payload, "NOTIFY") && !strings.Contains(payload, "M-SEARCH") {
return
}
f, err := os.OpenFile(ssdpFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return
}
defer f.Close()
srcIP := packet.NetworkLayer().NetworkFlow().Src().String()
dstIP := packet.NetworkLayer().NetworkFlow().Dst().String()
timestamp := packet.Metadata().Timestamp.Format("2006-01-02 15:04:05.000")
fmt.Fprintf(f, "[%s] %s:%d -> %s:%d\n", timestamp, srcIP, udp.SrcPort, dstIP, udp.DstPort)
fmt.Fprintf(f, "%s\n", strings.TrimSpace(payload))
fmt.Fprintln(f, "-------------------------------------------------")
}
func extractDNS(packet gopacket.Packet, dns *layers.DNS, dnsFile, mdnsFile *os.File) {
srcIP := packet.NetworkLayer().NetworkFlow().Src().String()
dstIP := packet.NetworkLayer().NetworkFlow().Dst().String()
isMDNS := false
if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil {
udp, _ := udpLayer.(*layers.UDP)
if udp.DstPort == 5353 || udp.SrcPort == 5353 {
isMDNS = true
}
}
out := dnsFile
if isMDNS {
out = mdnsFile
}
timestamp := packet.Metadata().Timestamp.Format("2006-01-02 15:04:05.000")
prefix := fmt.Sprintf("[%s] %s -> %s", timestamp, srcIP, dstIP)
for _, q := range dns.Questions {
fmt.Fprintf(out, "%s | QUERY: %s (%s)\n", prefix, string(q.Name), q.Type)
}
for _, a := range dns.Answers {
val := ""
if a.IP != nil {
val = a.IP.String()
} else if len(a.CNAME) > 0 {
val = string(a.CNAME)
} else if len(a.PTR) > 0 {
val = string(a.PTR)
} else if len(a.TXTs) > 0 {
var txts []string
for _, t := range a.TXTs {
txts = append(txts, string(t))
}
val = strings.Join(txts, " ")
} else {
val = fmt.Sprintf("Type: %s", a.Type)
}
fmt.Fprintf(out, "%s | ANSWER: %s -> %s\n", prefix, string(a.Name), val)
}
}
func extractWebSocket(packet gopacket.Packet, tcp *layers.TCP, filterIP string, wsFile *os.File) {
srcIP := packet.NetworkLayer().NetworkFlow().Src().String()
dstIP := packet.NetworkLayer().NetworkFlow().Dst().String()
if filterIP != "" && srcIP != filterIP && dstIP != filterIP {
return
}
payload := tcp.Payload
if len(payload) == 0 {
return
}
// Check for WebSocket Frame (Sliding search)
for i := 0; i < len(payload)-2; i++ {
firstByte := payload[i]
// Opcode 1 (Text) or 2 (Binary).
if (firstByte&0xF0) == 0x80 && (firstByte&0x0F == 1 || firstByte&0x0F == 2) {
secondByte := payload[i+1]
mask := (secondByte & 0x80) != 0
length := int(secondByte & 0x7F)
offset := i + 2
if length == 126 {
if len(payload) < offset+2 {
continue
}
length = int(payload[offset])<<8 | int(payload[offset+1])
offset += 2
} else if length == 127 {
if len(payload) < offset+8 {
continue
}
length = int(payload[offset+4])<<24 | int(payload[offset+5])<<16 | int(payload[offset+6])<<8 | int(payload[offset+7])
offset += 8
}
if mask {
if len(payload) < offset+4+length {
continue
}
maskKey := payload[offset : offset+4]
offset += 4
data := make([]byte, length)
for j := 0; j < length; j++ {
data[j] = payload[offset+j] ^ maskKey[j%4]
}
printInteraction(packet, tcp, data, wsFile)
i = offset + length - 1
} else {
if len(payload) >= offset+length {
data := payload[offset : offset+length]
printInteraction(packet, tcp, data, wsFile)
i = offset + length - 1
}
}
}
}
}
func printInteraction(packet gopacket.Packet, tcp *layers.TCP, data []byte, out io.Writer) {
src := packet.NetworkLayer().NetworkFlow().Src().String()
dst := packet.NetworkLayer().NetworkFlow().Dst().String()
fmt.Fprintf(out, "### WebSocket Message: %s -> %s\n", src, dst)
fmt.Fprintf(out, "// Timestamp: %s\n", packet.Metadata().Timestamp)
fmt.Fprintf(out, "// Ports: %d -> %d\n", tcp.SrcPort, tcp.DstPort)
fmt.Fprintln(out)
// Try to detect if it's GZIP
content := ""
if len(data) > 2 && data[0] == 0x1f && data[1] == 0x8b {
fmt.Fprintln(out, "// [Detected GZIP compression]")
content = decompressGzip(data)
} else {
content = string(data)
}
fmt.Fprintln(out, "/*")
fmt.Fprintln(out, strings.TrimSpace(content))
fmt.Fprintln(out, "*/")
fmt.Fprintln(out, "")
fmt.Fprintln(out, "-------------------------------------------------")
fmt.Fprintln(out, "")
}
func decompressGzip(data []byte) string {
b := bytes.NewBuffer(data)
r, err := gzip.NewReader(b)
if err != nil {
return "[Error: Failed to create GZIP reader: " + err.Error() + "]"
}
defer r.Close()
res, err := io.ReadAll(r)
if err != nil {
return "[Error: Failed to decompress GZIP: " + err.Error() + "]"
}
return string(res)
}
+32
View File
@@ -0,0 +1,32 @@
# HTTP Body Diff Tool
This tool extracts and compares response bodies from two `.http` files.
It supports XML and JSON normalization (pretty-printing) and automatically masks common "noisy" fields like timestamps to make actual differences easier to spot.
## Usage
```bash
go run scripts/http-diff/main.go <path/to/file1.http> <path/to/file2.http>
```
To generate a side-by-side HTML report:
```bash
go run scripts/http-diff/main.go --html report.html <path/to/file1.http> <path/to/file2.http>
```
## Features
- **Body Extraction**: Automatically finds the response body within the `/* ... */` comment block at the end of the file.
- **Side-by-Side View**: Generates an HTML report with a clear side-by-side comparison.
- **Normalization**:
- Pretty-prints XML and JSON.
- Trims whitespace from XML character data.
- **Noise Reduction**:
- Automatically replaces ISO 8601 timestamps with `[TIMESTAMP]`.
- Masks specific XML tags: `<updatedOn>`, `<createdOn>`, `<lastModified>`, `<timestamp>`.
- Masks specific JSON keys: `timestamp`, `updatedOn`, `createdOn`, `expires_at`.
- **Diff Output**:
- Displays a line-by-line diff.
- Show context for unchanged parts (first and last two lines, with `...` in between).
- Uses `+` for additions and `-` for deletions.
+299
View File
@@ -0,0 +1,299 @@
package main
import (
"bytes"
"encoding/json"
"encoding/xml"
"flag"
"fmt"
"html"
"io"
"os"
"regexp"
"strings"
"github.com/sergi/go-diff/diffmatchpatch"
)
func main() {
htmlOutput := flag.String("html", "", "Path to save HTML diff report")
flag.Parse()
args := flag.Args()
if len(args) < 2 {
fmt.Println("Usage: http-diff [options] <file1.http> <file2.http>")
fmt.Println("Options:")
flag.PrintDefaults()
os.Exit(1)
}
file1 := args[0]
file2 := args[1]
body1, err := extractBody(file1)
if err != nil {
fmt.Printf("Error extracting body from %s: %v\n", file1, err)
os.Exit(1)
}
body2, err := extractBody(file2)
if err != nil {
fmt.Printf("Error extracting body from %s: %v\n", file2, err)
os.Exit(1)
}
norm1 := normalize(body1)
norm2 := normalize(body2)
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(norm1, norm2, false)
lineDiffs := dmp.DiffCleanupSemantic(diffs)
if *htmlOutput != "" {
err := generateHTML(*htmlOutput, file1, file2, lineDiffs)
if err != nil {
fmt.Printf("Error generating HTML: %v\n", err)
os.Exit(1)
}
fmt.Printf("HTML report generated: %s\n", *htmlOutput)
return
}
// Custom line-by-line diff for better readability
for _, diff := range lineDiffs {
switch diff.Type {
case diffmatchpatch.DiffInsert:
lines := strings.Split(diff.Text, "\n")
for _, line := range lines {
if line != "" {
fmt.Printf("+ %s\n", line)
}
}
case diffmatchpatch.DiffDelete:
lines := strings.Split(diff.Text, "\n")
for _, line := range lines {
if line != "" {
fmt.Printf("- %s\n", line)
}
}
case diffmatchpatch.DiffEqual:
// Optionally skip unchanged lines or show context
lines := strings.Split(diff.Text, "\n")
// Filter out empty lines from splitting
var cleanLines []string
for _, l := range lines {
if strings.TrimSpace(l) != "" {
cleanLines = append(cleanLines, l)
}
}
if len(cleanLines) > 6 {
fmt.Printf(" %s\n", cleanLines[0])
fmt.Printf(" %s\n", cleanLines[1])
fmt.Printf(" ...\n")
fmt.Printf(" %s\n", cleanLines[len(cleanLines)-2])
fmt.Printf(" %s\n", cleanLines[len(cleanLines)-1])
} else {
for _, line := range cleanLines {
fmt.Printf(" %s\n", line)
}
}
}
}
}
func extractBody(path string) (string, error) {
content, err := os.ReadFile(path)
if err != nil {
return "", err
}
// Use a non-greedy regex to find the LAST /* ... */ block which typically contains the body
re := regexp.MustCompile(`(?s)/\*\s*(<\?xml.*?|\{.*?|\[.*?)\s*\*/`)
matches := re.FindAllStringSubmatch(string(content), -1)
if len(matches) > 0 {
// Return the last match as it's more likely to be the response body
lastMatch := matches[len(matches)-1]
return strings.TrimSpace(lastMatch[1]), nil
}
return "", fmt.Errorf("could not find body in /* ... */ block")
}
func normalize(body string) string {
body = strings.TrimSpace(body)
if body == "" {
return ""
}
// Mask common timestamps and changing fields
timestampRegex := regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})`)
body = timestampRegex.ReplaceAllString(body, "[TIMESTAMP]")
// Mask account IDs if they are variable, but usually they match in these files.
// Let's stick to timestamps for now.
// Try XML first
if strings.HasPrefix(body, "<?xml") || strings.Contains(body, "<") {
// For XML, let's also try to mask specific tags like <updatedOn> or <createdOn>
tagsToMask := []string{"updatedOn", "createdOn", "lastModified", "timestamp"}
for _, tag := range tagsToMask {
re := regexp.MustCompile(fmt.Sprintf(`<%s>.*?</%s>`, tag, tag))
body = re.ReplaceAllString(body, fmt.Sprintf("<%s>[MASKED]</%s>", tag, tag))
}
// Also mask empty attributes that might be noisy, like displayName=""
body = regexp.MustCompile(`\s+displayName=""`).ReplaceAllString(body, "")
var out bytes.Buffer
decoder := xml.NewDecoder(strings.NewReader(body))
encoder := xml.NewEncoder(&out)
encoder.Indent("", " ")
for {
token, err := decoder.Token()
if err == io.EOF {
break
}
if err != nil {
// If it's not valid XML, maybe it's just a fragment, continue or return body
break
}
// Trim whitespace from CharData to normalize
if cd, ok := token.(xml.CharData); ok {
token = xml.CharData(bytes.TrimSpace(cd))
}
err = encoder.EncodeToken(token)
if err != nil {
break
}
}
encoder.Flush()
if out.Len() > 0 {
return out.String()
}
}
// Try JSON
if strings.HasPrefix(body, "{") || strings.HasPrefix(body, "[") {
var obj interface{}
if err := json.Unmarshal([]byte(body), &obj); err == nil {
// Mask some JSON fields if they are common
maskJSON(obj)
pretty, _ := json.MarshalIndent(obj, "", " ")
return string(pretty)
}
}
return body
}
func maskJSON(data interface{}) {
switch v := data.(type) {
case map[string]interface{}:
for k, val := range v {
if strings.Contains(strings.ToLower(k), "timestamp") || k == "updatedOn" || k == "createdOn" || k == "expires_at" {
v[k] = "[MASKED]"
} else {
maskJSON(val)
}
}
case []interface{}:
for _, item := range v {
maskJSON(item)
}
}
}
func generateHTML(path, file1, file2 string, diffs []diffmatchpatch.Diff) error {
var sb strings.Builder
sb.WriteString(`<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>HTTP Diff Report</title>
<style>
body { font-family: monospace; line-height: 1.2; background: #f8f9fa; color: #212529; padding: 20px; }
.container { max-width: 1200px; margin: 0 auto; background: #fff; padding: 20px; border: 1px solid #dee2e6; border-radius: 4px; }
.header { margin-bottom: 20px; border-bottom: 2px solid #eee; padding-bottom: 10px; }
.diff-table { width: 100%; border-collapse: collapse; table-layout: fixed; }
.diff-table td { vertical-align: top; padding: 2px 4px; border: 1px solid #eee; overflow-wrap: break-word; }
.line-num { width: 40px; text-align: right; color: #999; background: #fdfdfd; user-select: none; }
.diff-equal { background: #fff; }
.diff-insert { background: #e6ffec; text-decoration: none; color: #1a7f37; }
.diff-delete { background: #ffebe9; text-decoration: none; color: #cf222e; }
.diff-change-marker { font-weight: bold; margin-right: 5px; }
h1 { font-size: 1.5rem; margin-bottom: 0.5rem; }
.files { font-size: 0.9rem; color: #666; }
pre { margin: 0; white-space: pre-wrap; }
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>HTTP Response Body Diff</h1>
<div class="files">
Left: <strong>` + html.EscapeString(file1) + `</strong><br>
Right: <strong>` + html.EscapeString(file2) + `</strong>
</div>
</div>
<table class="diff-table">
`)
type sideBySideLine struct {
leftText string
rightText string
class string
}
var lines []sideBySideLine
for _, diff := range diffs {
text := html.EscapeString(diff.Text)
split := strings.Split(text, "\n")
// Remove trailing empty string from split if it exists
if len(split) > 0 && split[len(split)-1] == "" {
split = split[:len(split)-1]
}
switch diff.Type {
case diffmatchpatch.DiffEqual:
for _, line := range split {
lines = append(lines, sideBySideLine{leftText: line, rightText: line, class: "diff-equal"})
}
case diffmatchpatch.DiffInsert:
for _, line := range split {
lines = append(lines, sideBySideLine{leftText: "", rightText: line, class: "diff-insert"})
}
case diffmatchpatch.DiffDelete:
for _, line := range split {
lines = append(lines, sideBySideLine{leftText: line, rightText: "", class: "diff-delete"})
}
}
}
for i, line := range lines {
leftMarker := ""
rightMarker := ""
if line.class == "diff-insert" {
rightMarker = "+"
} else if line.class == "diff-delete" {
leftMarker = "-"
}
sb.WriteString(fmt.Sprintf(`
<tr class="%s">
<td class="line-num">%d</td>
<td><pre><span class="diff-change-marker">%s</span>%s</pre></td>
<td class="line-num">%d</td>
<td><pre><span class="diff-change-marker">%s</span>%s</pre></td>
</tr>`, line.class, i+1, leftMarker, line.leftText, i+1, rightMarker, line.rightText))
}
sb.WriteString(`
</table>
</div>
</body>
</html>
`)
return os.WriteFile(path, []byte(sb.String()), 0644)
}